From 224ad82cc23531090602b5789f08c687d3be8c7a Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 24 Apr 2024 02:31:31 +1200 Subject: [PATCH 001/125] Remove old bake github pages workflow. --- gems.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/gems.rb b/gems.rb index e54c007..cbab6e7 100644 --- a/gems.rb +++ b/gems.rb @@ -24,7 +24,6 @@ gem "bake-modernize" gem "bake-gem" - gem "bake-github-pages" gem "utopia-project" end From c3759dab80559f49b30dc1e846c12f82376f1258 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 24 Apr 2024 02:51:23 +1200 Subject: [PATCH 002/125] Update dependencies. --- async-http.gemspec | 2 +- gems.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/async-http.gemspec b/async-http.gemspec index da7108c..7f2a9b9 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -26,7 +26,7 @@ Gem::Specification.new do |spec| spec.add_dependency "async", ">= 2.10.2" spec.add_dependency "async-pool", ">= 0.6.1" - spec.add_dependency "io-endpoint", "~> 0.10.0" + spec.add_dependency "io-endpoint", "~> 0.10.1" spec.add_dependency "io-stream", "~> 0.3.0" spec.add_dependency "protocol-http", "~> 0.26.0" spec.add_dependency "protocol-http1", "~> 0.19.0" diff --git a/gems.rb b/gems.rb index cbab6e7..dbbd8bd 100644 --- a/gems.rb +++ b/gems.rb @@ -24,6 +24,7 @@ gem "bake-modernize" gem "bake-gem" + gem "falcon", "~> 0.46" gem "utopia-project" end From 241a223fd855a6edb8ff50efa0ca93d27bd6af8c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 24 Apr 2024 16:49:09 +1200 Subject: [PATCH 003/125] Relax dependencies on `io-stream` and `io-endpoint`. --- async-http.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index 7f2a9b9..cf3ee87 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -26,8 +26,8 @@ Gem::Specification.new do |spec| spec.add_dependency "async", ">= 2.10.2" spec.add_dependency "async-pool", ">= 0.6.1" - spec.add_dependency "io-endpoint", "~> 0.10.1" - spec.add_dependency "io-stream", "~> 0.3.0" + spec.add_dependency "io-endpoint", "~> 0.10" + spec.add_dependency "io-stream", "~> 0.4" spec.add_dependency "protocol-http", "~> 0.26.0" spec.add_dependency "protocol-http1", "~> 0.19.0" spec.add_dependency "protocol-http2", "~> 0.17.0" From 5f5eb26aee6187c5d200767df6a8c1b57c0e9573 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 24 Apr 2024 16:51:34 +1200 Subject: [PATCH 004/125] Remove invalid require `async/io/ssl_endpoint`. --- test/async/http/client.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/async/http/client.rb b/test/async/http/client.rb index 040b82a..621364f 100644 --- a/test/async/http/client.rb +++ b/test/async/http/client.rb @@ -7,7 +7,6 @@ require 'async/http/client' require 'async/reactor' -require 'async/io/ssl_socket' require 'async/http/endpoint' require 'protocol/http/accept_encoding' From 5e38b2a059b5162ced4a95725592b35fe4e14ece Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 24 Apr 2024 17:20:40 +1200 Subject: [PATCH 005/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index bed111d..21133b6 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.65.0" + VERSION = "0.65.1" end end From c12f46273f7c474628ce0df55476c7e2bdcc73da Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 30 Apr 2024 13:16:14 +1200 Subject: [PATCH 006/125] Let's be more specific w.r.t. `Async::HTTP::Server#run`. (#158) --- lib/async/http/server.rb | 10 ++++++++-- test/async/http/server.rb | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 test/async/http/server.rb diff --git a/lib/async/http/server.rb b/lib/async/http/server.rb index b7cde94..c1c5d9c 100755 --- a/lib/async/http/server.rb +++ b/lib/async/http/server.rb @@ -64,9 +64,15 @@ def accept(peer, address, task: Task.current) ensure connection&.close end - + + # @returns [Array(Async::Task)] The task that is running the server. def run - @endpoint.accept(&self.method(:accept)) + Async do + @endpoint.accept(&self.method(:accept)) + + # Wait for all children to finish: + self.children.each(&:wait) + end end Traces::Provider(self) do diff --git a/test/async/http/server.rb b/test/async/http/server.rb new file mode 100644 index 0000000..68dd577 --- /dev/null +++ b/test/async/http/server.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2017-2024, by Samuel Williams. + +require 'async/http/server' +require 'async/http/endpoint' +require 'sus/fixtures/async' + +describe Async::HTTP::Server do + include Sus::Fixtures::Async::ReactorContext + + let(:endpoint) {Async::HTTP::Endpoint.parse('http://localhost:0')} + let(:app) {Protocol::HTTP::Middleware::Okay} + let(:server) {subject.new(app, endpoint)} + + with '#run' do + it "runs the server" do + task = server.run + + expect(task).to be_a(Async::Task) + + task.stop + end + end +end From 4607e9d9e490bc605bb5dc3f4d2a2b5683b6ca1f Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 30 Apr 2024 13:17:10 +1200 Subject: [PATCH 007/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 21133b6..7a666ce 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.65.1" + VERSION = "0.66.0" end end From 786d9d910bf76a1946623904064438c42c85a771 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 30 Apr 2024 13:29:38 +1200 Subject: [PATCH 008/125] Fix incorrect `self` -> `task`. --- lib/async/http/server.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/async/http/server.rb b/lib/async/http/server.rb index c1c5d9c..1f6f4a7 100755 --- a/lib/async/http/server.rb +++ b/lib/async/http/server.rb @@ -67,11 +67,11 @@ def accept(peer, address, task: Task.current) # @returns [Array(Async::Task)] The task that is running the server. def run - Async do + Async do |task| @endpoint.accept(&self.method(:accept)) # Wait for all children to finish: - self.children.each(&:wait) + task.children.each(&:wait) end end From 9b034a87e298c45d0c9250a83862c645c9ad1970 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 30 Apr 2024 13:29:51 +1200 Subject: [PATCH 009/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 7a666ce..bfbcafe 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.66.0" + VERSION = "0.66.1" end end From 6b8d2b51409a73a24896871903f734b965cc0161 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 30 Apr 2024 13:35:55 +1200 Subject: [PATCH 010/125] Add missing require for `Async`. --- lib/async/http/server.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/async/http/server.rb b/lib/async/http/server.rb index 1f6f4a7..0d4790c 100755 --- a/lib/async/http/server.rb +++ b/lib/async/http/server.rb @@ -4,10 +4,9 @@ # Copyright, 2017-2024, by Samuel Williams. # Copyright, 2019, by Brian Morearty. +require 'async' require 'io/endpoint' - require 'protocol/http/middleware' - require 'traces/provider' require_relative 'protocol' From 827a0a8b1ccc88a8cd37acaa3b9bdd10f288d38e Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 30 Apr 2024 13:36:02 +1200 Subject: [PATCH 011/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index bfbcafe..ba32eb3 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.66.1" + VERSION = "0.66.2" end end From 28698fae054581a0ab071f091552edb58f623ec3 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 10 May 2024 17:01:35 +0900 Subject: [PATCH 012/125] Bump dependencies. --- async-http.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/async-http.gemspec b/async-http.gemspec index cf3ee87..7c9c85b 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -26,7 +26,7 @@ Gem::Specification.new do |spec| spec.add_dependency "async", ">= 2.10.2" spec.add_dependency "async-pool", ">= 0.6.1" - spec.add_dependency "io-endpoint", "~> 0.10" + spec.add_dependency "io-endpoint", "~> 0.10", ">= 0.10.3" spec.add_dependency "io-stream", "~> 0.4" spec.add_dependency "protocol-http", "~> 0.26.0" spec.add_dependency "protocol-http1", "~> 0.19.0" From 019310e275b7b892e297b74d67cb09e33414bc59 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 10 May 2024 17:01:40 +0900 Subject: [PATCH 013/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index ba32eb3..19c1201 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.66.2" + VERSION = "0.66.3" end end From 1e4fc00d5dc1831aad16d3431264a712cad376cd Mon Sep 17 00:00:00 2001 From: zarqman Date: Sun, 9 Jun 2024 08:19:03 -0600 Subject: [PATCH 014/125] Add auto-detection of HTTP1 vs HTTP2 for inbound connections. (#128) --- lib/async/http/endpoint.rb | 4 +-- lib/async/http/protocol/http.rb | 55 +++++++++++++++++++++++++++++++ lib/async/http/protocol/http1.rb | 7 ++-- lib/async/http/protocol/http10.rb | 5 +-- lib/async/http/protocol/http11.rb | 5 +-- lib/async/http/protocol/http2.rb | 7 ++-- test/async/http/endpoint.rb | 2 +- test/async/http/protocol/http.rb | 37 +++++++++++++++++++++ 8 files changed, 109 insertions(+), 13 deletions(-) create mode 100644 lib/async/http/protocol/http.rb create mode 100755 test/async/http/protocol/http.rb diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index 995d2f9..f4b1651 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -8,7 +8,7 @@ require 'io/endpoint/host_endpoint' require 'io/endpoint/ssl_endpoint' -require_relative 'protocol/http1' +require_relative 'protocol/http' require_relative 'protocol/https' module Async @@ -84,7 +84,7 @@ def protocol if secure? Protocol::HTTPS else - Protocol::HTTP1 + Protocol::HTTP end end end diff --git a/lib/async/http/protocol/http.rb b/lib/async/http/protocol/http.rb new file mode 100644 index 0000000..c37616a --- /dev/null +++ b/lib/async/http/protocol/http.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2023, by Thomas Morgan. + +require_relative 'http1' +require_relative 'http2' + +module Async + module HTTP + module Protocol + # HTTP is an http:// server that auto-selects HTTP/1.1 or HTTP/2 by detecting the HTTP/2 + # connection preface. + module HTTP + HTTP2_PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + HTTP2_PREFACE_SIZE = HTTP2_PREFACE.bytesize + + def self.protocol_for(stream) + # Detect HTTP/2 connection preface + # https://www.rfc-editor.org/rfc/rfc9113.html#section-3.4 + preface = stream.peek do |read_buffer| + if read_buffer.bytesize >= HTTP2_PREFACE_SIZE + break read_buffer[0, HTTP2_PREFACE_SIZE] + elsif read_buffer.bytesize > 0 + # If partial read_buffer already doesn't match, no need to wait for more bytes. + break read_buffer unless HTTP2_PREFACE[read_buffer] + end + end + + if preface == HTTP2_PREFACE + HTTP2 + else + HTTP1 + end + end + + # Only inbound connections can detect HTTP1 vs HTTP2 for http://. + # Outbound connections default to HTTP1. + def self.client(peer, **options) + HTTP1.client(peer, **options) + end + + def self.server(peer, **options) + stream = ::IO::Stream(peer) + + return protocol_for(stream).server(stream, **options) + end + + def self.names + ["h2", "http/1.1", "http/1.0"] + end + end + end + end +end diff --git a/lib/async/http/protocol/http1.rb b/lib/async/http/protocol/http1.rb index e30f341..e4024db 100644 --- a/lib/async/http/protocol/http1.rb +++ b/lib/async/http/protocol/http1.rb @@ -2,11 +2,12 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. +# Copyright, 2023, by Thomas Morgan. require_relative 'http1/client' require_relative 'http1/server' -require 'io/stream/buffered' +require 'io/stream' module Async module HTTP @@ -23,13 +24,13 @@ def self.trailer? end def self.client(peer) - stream = ::IO::Stream::Buffered.wrap(peer) + stream = ::IO::Stream(peer) return HTTP1::Client.new(stream, VERSION) end def self.server(peer) - stream = ::IO::Stream::Buffered.wrap(peer) + stream = ::IO::Stream(peer) return HTTP1::Server.new(stream, VERSION) end diff --git a/lib/async/http/protocol/http10.rb b/lib/async/http/protocol/http10.rb index 9066d69..5d5e9b3 100755 --- a/lib/async/http/protocol/http10.rb +++ b/lib/async/http/protocol/http10.rb @@ -2,6 +2,7 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. +# Copyright, 2023, by Thomas Morgan. require_relative 'http1' @@ -20,13 +21,13 @@ def self.trailer? end def self.client(peer) - stream = ::IO::Stream::Buffered.wrap(peer) + stream = ::IO::Stream(peer) return HTTP1::Client.new(stream, VERSION) end def self.server(peer) - stream = ::IO::Stream::Buffered.wrap(peer) + stream = ::IO::Stream(peer) return HTTP1::Server.new(stream, VERSION) end diff --git a/lib/async/http/protocol/http11.rb b/lib/async/http/protocol/http11.rb index 083dc14..46a3762 100644 --- a/lib/async/http/protocol/http11.rb +++ b/lib/async/http/protocol/http11.rb @@ -3,6 +3,7 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. # Copyright, 2018, by Janko Marohnić. +# Copyright, 2023, by Thomas Morgan. require_relative 'http1' @@ -21,13 +22,13 @@ def self.trailer? end def self.client(peer) - stream = ::IO::Stream::Buffered.wrap(peer) + stream = ::IO::Stream(peer) return HTTP1::Client.new(stream, VERSION) end def self.server(peer) - stream = ::IO::Stream::Buffered.wrap(peer) + stream = ::IO::Stream(peer) return HTTP1::Server.new(stream, VERSION) end diff --git a/lib/async/http/protocol/http2.rb b/lib/async/http/protocol/http2.rb index 7a75a34..bc560d7 100644 --- a/lib/async/http/protocol/http2.rb +++ b/lib/async/http/protocol/http2.rb @@ -2,11 +2,12 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2023, by Thomas Morgan. require_relative 'http2/client' require_relative 'http2/server' -require 'io/stream/buffered' +require 'io/stream' module Async module HTTP @@ -37,7 +38,7 @@ def self.trailer? } def self.client(peer, settings = CLIENT_SETTINGS) - stream = ::IO::Stream::Buffered.wrap(peer) + stream = ::IO::Stream(peer) client = Client.new(stream) client.send_connection_preface(settings) @@ -47,7 +48,7 @@ def self.client(peer, settings = CLIENT_SETTINGS) end def self.server(peer, settings = SERVER_SETTINGS) - stream = ::IO::Stream::Buffered.wrap(peer) + stream = ::IO::Stream(peer) server = Server.new(stream) server.read_connection_preface(settings) diff --git a/test/async/http/endpoint.rb b/test/async/http/endpoint.rb index e04169b..c16f4ef 100644 --- a/test/async/http/endpoint.rb +++ b/test/async/http/endpoint.rb @@ -155,7 +155,7 @@ describe Async::HTTP::Endpoint.parse("http://www.google.com/search") do it "should select the correct protocol" do - expect(subject.protocol).to be == Async::HTTP::Protocol::HTTP1 + expect(subject.protocol).to be == Async::HTTP::Protocol::HTTP end it "should parse the correct hostname" do diff --git a/test/async/http/protocol/http.rb b/test/async/http/protocol/http.rb new file mode 100755 index 0000000..0de6e7c --- /dev/null +++ b/test/async/http/protocol/http.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2023, by Thomas Morgan. + +require 'async/http/protocol/http' +require 'async/http/a_protocol' + +describe Async::HTTP::Protocol::HTTP do + with 'server' do + include Sus::Fixtures::Async::HTTP::ServerContext + let(:protocol) {subject} + + with 'http11 client' do + it 'should make a successful request' do + response = client.get('/') + expect(response).to be(:success?) + expect(response.version).to be == 'HTTP/1.1' + response.read + end + end + + with 'http2 client' do + def make_client(endpoint, **options) + options[:protocol] = Async::HTTP::Protocol::HTTP2 + super + end + + it 'should make a successful request' do + response = client.get('/') + expect(response).to be(:success?) + expect(response.version).to be == 'HTTP/2' + response.read + end + end + end +end From 722cd78ece5933328e98767ea300f9f860a11b44 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 9 Jun 2024 23:47:38 +0900 Subject: [PATCH 015/125] Add example of plaintext HTTP/1 and HTTP/2. --- examples/hello/config.ru | 16 +++++++ examples/hello/gems.locked | 85 ++++++++++++++++++++++++++++++++++++++ examples/hello/gems.rb | 4 ++ examples/hello/readme.md | 63 ++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+) create mode 100755 examples/hello/config.ru create mode 100644 examples/hello/gems.locked create mode 100644 examples/hello/gems.rb create mode 100644 examples/hello/readme.md diff --git a/examples/hello/config.ru b/examples/hello/config.ru new file mode 100755 index 0000000..7179e56 --- /dev/null +++ b/examples/hello/config.ru @@ -0,0 +1,16 @@ +#!/usr/bin/env falcon --verbose serve -c +# frozen_string_literal: true + +require 'async' +require 'async/barrier' +require 'net/http' +require 'uri' + +run do |env| + i = 1_000_000 + while i > 0 + i -= 1 + end + + [200, {}, ["Hello World!"]] +end diff --git a/examples/hello/gems.locked b/examples/hello/gems.locked new file mode 100644 index 0000000..9b51516 --- /dev/null +++ b/examples/hello/gems.locked @@ -0,0 +1,85 @@ +PATH + remote: ../.. + specs: + async-http (0.66.3) + async (>= 2.10.2) + async-pool (>= 0.6.1) + io-endpoint (~> 0.10, >= 0.10.3) + io-stream (~> 0.4) + protocol-http (~> 0.26.0) + protocol-http1 (~> 0.19.0) + protocol-http2 (~> 0.17.0) + traces (>= 0.10.0) + +GEM + remote: https://rubygems.org/ + specs: + async (2.12.0) + console (~> 1.25, >= 1.25.2) + fiber-annotation + io-event (~> 1.6) + async-container (0.18.2) + async (~> 2.10) + async-http-cache (0.4.3) + async-http (~> 0.56) + async-pool (0.6.1) + async (>= 1.25) + async-service (0.12.0) + async + async-container (~> 0.16) + console (1.25.2) + fiber-annotation + fiber-local (~> 1.1) + json + falcon (0.47.6) + async + async-container (~> 0.18) + async-http (~> 0.66, >= 0.66.3) + async-http-cache (~> 0.4.0) + async-service (~> 0.10) + bundler + localhost (~> 1.1) + openssl (~> 3.0) + process-metrics (~> 0.2.0) + protocol-rack (~> 0.5) + samovar (~> 2.3) + fiber-annotation (0.2.0) + fiber-local (1.1.0) + fiber-storage + fiber-storage (0.1.1) + io-endpoint (0.10.3) + io-event (1.6.0) + io-stream (0.4.0) + json (2.7.2) + localhost (1.3.1) + mapping (1.1.1) + openssl (3.2.0) + process-metrics (0.2.1) + console (~> 1.8) + samovar (~> 2.1) + protocol-hpack (1.4.3) + protocol-http (0.26.5) + protocol-http1 (0.19.1) + protocol-http (~> 0.22) + protocol-http2 (0.17.0) + protocol-hpack (~> 1.4) + protocol-http (~> 0.18) + protocol-rack (0.5.1) + protocol-http (~> 0.23) + rack (>= 1.0) + rack (3.0.11) + samovar (2.3.0) + console (~> 1.0) + mapping (~> 1.0) + traces (0.11.1) + +PLATFORMS + arm64-darwin-23 + ruby + +DEPENDENCIES + async-http! + falcon + +BUNDLED WITH + 2.5.9 diff --git a/examples/hello/gems.rb b/examples/hello/gems.rb new file mode 100644 index 0000000..b588426 --- /dev/null +++ b/examples/hello/gems.rb @@ -0,0 +1,4 @@ +source "https://rubygems.org" + +gem "async-http", path: "../../" +gem "falcon" diff --git a/examples/hello/readme.md b/examples/hello/readme.md new file mode 100644 index 0000000..a6410ff --- /dev/null +++ b/examples/hello/readme.md @@ -0,0 +1,63 @@ +# Hello Example + +## Server + +```bash +$ bundle update +$ bundle exec falcon serve --bind http://localhost:3000 +``` + +## Client + +### HTTP/1 + +```bash +$ curl -v http://localhost:3000 +* Host localhost:3000 was resolved. +* IPv6: ::1 +* IPv4: 127.0.0.1 +* Trying [::1]:3000... +* Connected to localhost (::1) port 3000 +> GET / HTTP/1.1 +> Host: localhost:3000 +> User-Agent: curl/8.7.1 +> Accept: */* +> +* Request completely sent off +< HTTP/1.1 200 OK +< vary: accept-encoding +< content-length: 12 +< +* Connection #0 to host localhost left intact +Hello World!⏎ +``` + +### HTTP/2 + +```bash +$ curl -v --http2-prior-knowledge http://localhost:3000 +* Host localhost:3000 was resolved. +* IPv6: ::1 +* IPv4: 127.0.0.1 +* Trying [::1]:3000... +* Connected to localhost (::1) port 3000 +* [HTTP/2] [1] OPENED stream for http://localhost:3000/ +* [HTTP/2] [1] [:method: GET] +* [HTTP/2] [1] [:scheme: http] +* [HTTP/2] [1] [:authority: localhost:3000] +* [HTTP/2] [1] [:path: /] +* [HTTP/2] [1] [user-agent: curl/8.7.1] +* [HTTP/2] [1] [accept: */*] +> GET / HTTP/2 +> Host: localhost:3000 +> User-Agent: curl/8.7.1 +> Accept: */* +> +* Request completely sent off +< HTTP/2 200 +< content-length: 12 +< vary: accept-encoding +< +* Connection #0 to host localhost left intact +Hello World!⏎ +``` From 4fc21027bff03029216409703d2400277b62c5aa Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 9 Jun 2024 23:49:45 +0900 Subject: [PATCH 016/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 19c1201..d7538c7 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.66.3" + VERSION = "0.67.0" end end From 53d5b4fc1117d9b9f3edecc467557c2e9489de93 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 10 Jun 2024 14:58:14 +0900 Subject: [PATCH 017/125] Add delay example. --- examples/delay/client.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 examples/delay/client.rb diff --git a/examples/delay/client.rb b/examples/delay/client.rb new file mode 100755 index 0000000..596c248 --- /dev/null +++ b/examples/delay/client.rb @@ -0,0 +1,28 @@ +#!/usr/bin/env ruby + +require "net/http" +require "async" +require "async/http/internet" +require "async/barrier" +require "async/semaphore" + +N_TIMES = 1000 + +Async do |task| + internet = Async::HTTP::Internet.new + barrier = Async::Barrier.new + + results = N_TIMES.times.map do |i| + barrier.async do + puts "Run #{i}" + + begin + response = internet.get("https://httpbin.org/delay/0.5") + ensure + response&.finish + end + end + end + + barrier.wait +end From 5b7bf2efd0e084bca0676c26df1bb49586c72b8e Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 10 Jun 2024 14:58:29 +0900 Subject: [PATCH 018/125] Update dependency on `protocol-http2`. --- async-http.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/async-http.gemspec b/async-http.gemspec index 7c9c85b..a118193 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -30,6 +30,6 @@ Gem::Specification.new do |spec| spec.add_dependency "io-stream", "~> 0.4" spec.add_dependency "protocol-http", "~> 0.26.0" spec.add_dependency "protocol-http1", "~> 0.19.0" - spec.add_dependency "protocol-http2", "~> 0.17.0" + spec.add_dependency "protocol-http2", "~> 0.18.0" spec.add_dependency "traces", ">= 0.10.0" end From d0894a0e1c9d7af40cbf2fa82716dea8b422c4ba Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 10 Jun 2024 14:58:43 +0900 Subject: [PATCH 019/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index d7538c7..28ac63e 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.67.0" + VERSION = "0.67.1" end end From 80dcbbe571d2dbb935389d8517b6abdf9127781c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 23 Jun 2024 14:02:34 +0900 Subject: [PATCH 020/125] Relax dependencies. --- async-http.gemspec | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index a118193..9609c61 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -25,11 +25,11 @@ Gem::Specification.new do |spec| spec.required_ruby_version = ">= 3.1" spec.add_dependency "async", ">= 2.10.2" - spec.add_dependency "async-pool", ">= 0.6.1" - spec.add_dependency "io-endpoint", "~> 0.10", ">= 0.10.3" + spec.add_dependency "async-pool", "~> 0.7" + spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" - spec.add_dependency "protocol-http", "~> 0.26.0" - spec.add_dependency "protocol-http1", "~> 0.19.0" - spec.add_dependency "protocol-http2", "~> 0.18.0" - spec.add_dependency "traces", ">= 0.10.0" + spec.add_dependency "protocol-http", "~> 0.26" + spec.add_dependency "protocol-http1", "~> 0.19" + spec.add_dependency "protocol-http2", "~> 0.18" + spec.add_dependency "traces", ">= 0.10" end From 63409fa65b5d6f9db65f983ec1db15abffc6e735 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 23 Jun 2024 14:04:21 +0900 Subject: [PATCH 021/125] Modernize gem. --- .github/workflows/documentation.yaml | 4 ++-- examples/delay/client.rb | 4 ++++ examples/hello/gems.rb | 5 +++++ lib/async/http/endpoint.rb | 1 + lib/async/http/protocol/http.rb | 2 +- lib/async/http/protocol/http1.rb | 2 +- lib/async/http/protocol/http10.rb | 2 +- lib/async/http/protocol/http11.rb | 2 +- lib/async/http/protocol/http2.rb | 2 +- license.md | 2 +- test/async/http/endpoint.rb | 1 + test/async/http/protocol/http.rb | 2 +- test/async/http/server.rb | 2 +- 13 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index 8dc5227..f5f553a 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -40,7 +40,7 @@ jobs: run: bundle exec bake utopia:project:static --force no - name: Upload documentation artifact - uses: actions/upload-pages-artifact@v2 + uses: actions/upload-pages-artifact@v3 with: path: docs @@ -55,4 +55,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v3 + uses: actions/deploy-pages@v4 diff --git a/examples/delay/client.rb b/examples/delay/client.rb index 596c248..d64b485 100755 --- a/examples/delay/client.rb +++ b/examples/delay/client.rb @@ -1,4 +1,8 @@ #!/usr/bin/env ruby +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2024, by Samuel Williams. require "net/http" require "async" diff --git a/examples/hello/gems.rb b/examples/hello/gems.rb index b588426..a5ad216 100644 --- a/examples/hello/gems.rb +++ b/examples/hello/gems.rb @@ -1,3 +1,8 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2024, by Samuel Williams. + source "https://rubygems.org" gem "async-http", path: "../../" diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index f4b1651..d975c8e 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -3,6 +3,7 @@ # Released under the MIT License. # Copyright, 2019-2024, by Samuel Williams. # Copyright, 2021-2022, by Adam Daniels. +# Copyright, 2024, by Thomas Morgan. require 'io/endpoint' require 'io/endpoint/host_endpoint' diff --git a/lib/async/http/protocol/http.rb b/lib/async/http/protocol/http.rb index c37616a..eb7d58b 100644 --- a/lib/async/http/protocol/http.rb +++ b/lib/async/http/protocol/http.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2023, by Thomas Morgan. +# Copyright, 2024, by Thomas Morgan. require_relative 'http1' require_relative 'http2' diff --git a/lib/async/http/protocol/http1.rb b/lib/async/http/protocol/http1.rb index e4024db..e51b57b 100644 --- a/lib/async/http/protocol/http1.rb +++ b/lib/async/http/protocol/http1.rb @@ -2,7 +2,7 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. -# Copyright, 2023, by Thomas Morgan. +# Copyright, 2024, by Thomas Morgan. require_relative 'http1/client' require_relative 'http1/server' diff --git a/lib/async/http/protocol/http10.rb b/lib/async/http/protocol/http10.rb index 5d5e9b3..793048a 100755 --- a/lib/async/http/protocol/http10.rb +++ b/lib/async/http/protocol/http10.rb @@ -2,7 +2,7 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. -# Copyright, 2023, by Thomas Morgan. +# Copyright, 2024, by Thomas Morgan. require_relative 'http1' diff --git a/lib/async/http/protocol/http11.rb b/lib/async/http/protocol/http11.rb index 46a3762..dd8135f 100644 --- a/lib/async/http/protocol/http11.rb +++ b/lib/async/http/protocol/http11.rb @@ -3,7 +3,7 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. # Copyright, 2018, by Janko Marohnić. -# Copyright, 2023, by Thomas Morgan. +# Copyright, 2024, by Thomas Morgan. require_relative 'http1' diff --git a/lib/async/http/protocol/http2.rb b/lib/async/http/protocol/http2.rb index bc560d7..96da7e2 100644 --- a/lib/async/http/protocol/http2.rb +++ b/lib/async/http/protocol/http2.rb @@ -2,7 +2,7 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -# Copyright, 2023, by Thomas Morgan. +# Copyright, 2024, by Thomas Morgan. require_relative 'http2/client' require_relative 'http2/server' diff --git a/license.md b/license.md index c694bcc..598f6a1 100644 --- a/license.md +++ b/license.md @@ -17,7 +17,7 @@ Copyright, 2021-2022, by Adam Daniels. Copyright, 2022, by Ian Ker-Seymer. Copyright, 2022, by Marco Concetto Rudilosso. Copyright, 2022, by Tim Meusel. -Copyright, 2023, by Thomas Morgan. +Copyright, 2023-2024, by Thomas Morgan. Copyright, 2023, by dependabot[bot]. Copyright, 2023, by Josh Huber. Copyright, 2024, by Anton Zhuravsky. diff --git a/test/async/http/endpoint.rb b/test/async/http/endpoint.rb index c16f4ef..9c501bd 100644 --- a/test/async/http/endpoint.rb +++ b/test/async/http/endpoint.rb @@ -3,6 +3,7 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2021-2022, by Adam Daniels. +# Copyright, 2024, by Thomas Morgan. require 'async/http/endpoint' diff --git a/test/async/http/protocol/http.rb b/test/async/http/protocol/http.rb index 0de6e7c..d787dc9 100755 --- a/test/async/http/protocol/http.rb +++ b/test/async/http/protocol/http.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2023, by Thomas Morgan. +# Copyright, 2024, by Thomas Morgan. require 'async/http/protocol/http' require 'async/http/a_protocol' diff --git a/test/async/http/server.rb b/test/async/http/server.rb index 68dd577..1067d8f 100644 --- a/test/async/http/server.rb +++ b/test/async/http/server.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2024, by Samuel Williams. +# Copyright, 2024, by Samuel Williams. require 'async/http/server' require 'async/http/endpoint' From 3a96b3aada48674078518760bb2db779ee95ae0d Mon Sep 17 00:00:00 2001 From: Igor Sidorov Date: Sun, 23 Jun 2024 08:59:01 +0300 Subject: [PATCH 022/125] Allow using custom endpoints for `Async::HTTP::Internet`. (#51) --- lib/async/http/endpoint.rb | 10 ++++++++++ lib/async/http/internet.rb | 4 ++-- test/async/http/internet.rb | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index d975c8e..02560e9 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -33,6 +33,16 @@ def self.for(scheme, hostname, path = "/", **options) ) end + # Coerce the given object into an endpoint. + # @parameter url [String | Endpoint] The URL or endpoint to convert. + def self.[](url) + if url.is_a?(Endpoint) + return url + else + Endpoint.parse(url.to_str) + end + end + # @option scheme [String] the scheme to use, overrides the URL scheme. # @option hostname [String] the hostname to connect to (or bind to), overrides the URL hostname (used for SNI). # @option port [Integer] the port to bind to, overrides the URL port. diff --git a/lib/async/http/internet.rb b/lib/async/http/internet.rb index f744084..a488f3f 100644 --- a/lib/async/http/internet.rb +++ b/lib/async/http/internet.rb @@ -39,7 +39,7 @@ def client_for(endpoint) # @parameter headers [Hash | Protocol::HTTP::Headers] The headers to send with the request. # @parameter body [String | Protocol::HTTP::Body] The body to send with the request. def call(method, url, headers = nil, body = nil) - endpoint = Endpoint.parse(url) + endpoint = Endpoint[url] client = self.client_for(endpoint) body = Body::Buffered.wrap(body) @@ -60,7 +60,7 @@ def close ::Protocol::HTTP::Methods.each do |name, verb| define_method(verb.downcase) do |url, headers = nil, body = nil| - self.call(verb, url.to_str, headers, body) + self.call(verb, url, headers, body) end end diff --git a/test/async/http/internet.rb b/test/async/http/internet.rb index ac25f40..dc8c064 100644 --- a/test/async/http/internet.rb +++ b/test/async/http/internet.rb @@ -33,4 +33,18 @@ expect(response).to be(:success?) expect{JSON.parse(response.read)}.not.to raise_exception end + + it 'can fetch remote website when given custom endpoint instead of url' do + ssl_context = OpenSSL::SSL::SSLContext.new + ssl_context.set_params(verify_mode: OpenSSL::SSL::VERIFY_NONE) + + # example of site with invalid certificate that will fail to be fetched without custom SSL options + endpoint = Async::HTTP::Endpoint.parse('https://expired.badssl.com', ssl_context: ssl_context) + + response = internet.get(endpoint, headers) + + expect(response).to be(:success?) + ensure + response&.close + end end From 2da8b15e45263cc97c64724124b3ddc742dc30db Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 23 Jun 2024 14:59:46 +0900 Subject: [PATCH 023/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 28ac63e..062ec90 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.67.1" + VERSION = "0.68.0" end end From 0f118111e22794a08101a6d0321bfff78175c51d Mon Sep 17 00:00:00 2001 From: Postmodern Date: Sun, 23 Jun 2024 00:53:26 -0700 Subject: [PATCH 024/125] Allow `Async::HTTP::Internet` methods to accept `URI::HTTP` objects. (#118) * `URI::HTTP` objects do not define a `to_str` method, so use `to_s` instead. --- lib/async/http/endpoint.rb | 2 +- test/async/http/internet.rb | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index 02560e9..da8618b 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -39,7 +39,7 @@ def self.[](url) if url.is_a?(Endpoint) return url else - Endpoint.parse(url.to_str) + Endpoint.parse(url.to_s) end end diff --git a/test/async/http/internet.rb b/test/async/http/internet.rb index dc8c064..0eaf42a 100644 --- a/test/async/http/internet.rb +++ b/test/async/http/internet.rb @@ -23,6 +23,15 @@ response.close end + it "can accept URI::HTTP objects" do + uri = URI.parse("https://www.codeotaku.com/index") + response = internet.get(uri, headers) + + expect(response).to be(:success?) + ensure + response&.close + end + let(:sample) {{"hello" => "world"}} let(:body) {[JSON.dump(sample)]} From 1591f3f74c5a5a66cfb706e444cdba0d81856f3d Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 23 Jun 2024 17:17:36 +0900 Subject: [PATCH 025/125] Update copyrights. --- .mailmap | 1 + async-http.gemspec | 2 +- lib/async/http/endpoint.rb | 2 ++ lib/async/http/internet.rb | 3 ++- lib/async/http/protocol/http.rb | 1 + license.md | 3 ++- test/async/http/internet.rb | 4 +++- test/async/http/protocol/http.rb | 1 + 8 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.mailmap b/.mailmap index 50b4e97..c42653c 100644 --- a/.mailmap +++ b/.mailmap @@ -1,3 +1,4 @@ Viacheslav Koval Sam Shadwell Thomas Morgan +Hal Brodigan diff --git a/async-http.gemspec b/async-http.gemspec index 9609c61..f34cd30 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -7,7 +7,7 @@ Gem::Specification.new do |spec| spec.version = Async::HTTP::VERSION spec.summary = "A HTTP client and server library." - spec.authors = ["Samuel Williams", "Brian Morearty", "Bruno Sutic", "Janko Marohnić", "Thomas Morgan", "Adam Daniels", "Anton Zhuravsky", "Cyril Roelandt", "Denis Talakevich", "Ian Ker-Seymer", "Igor Sidorov", "Josh Huber", "Marco Concetto Rudilosso", "Olle Jonsson", "Orgad Shaneh", "Sam Shadwell", "Stefan Wrobel", "Tim Meusel", "Trevor Turk", "Viacheslav Koval", "dependabot[bot]"] + spec.authors = ["Samuel Williams", "Brian Morearty", "Bruno Sutic", "Janko Marohnić", "Thomas Morgan", "Adam Daniels", "Igor Sidorov", "Anton Zhuravsky", "Cyril Roelandt", "Denis Talakevich", "Hal Brodigan", "Ian Ker-Seymer", "Josh Huber", "Marco Concetto Rudilosso", "Olle Jonsson", "Orgad Shaneh", "Sam Shadwell", "Stefan Wrobel", "Tim Meusel", "Trevor Turk", "Viacheslav Koval", "dependabot[bot]"] spec.license = "MIT" spec.cert_chain = ['release.cert'] diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index da8618b..dd80f9c 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -4,6 +4,8 @@ # Copyright, 2019-2024, by Samuel Williams. # Copyright, 2021-2022, by Adam Daniels. # Copyright, 2024, by Thomas Morgan. +# Copyright, 2024, by Igor Sidorov. +# Copyright, 2024, by Hal Brodigan. require 'io/endpoint' require 'io/endpoint/host_endpoint' diff --git a/lib/async/http/internet.rb b/lib/async/http/internet.rb index a488f3f..37fd379 100644 --- a/lib/async/http/internet.rb +++ b/lib/async/http/internet.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2024, by Igor Sidorov. require_relative 'client' require_relative 'endpoint' diff --git a/lib/async/http/protocol/http.rb b/lib/async/http/protocol/http.rb index eb7d58b..6b57c08 100644 --- a/lib/async/http/protocol/http.rb +++ b/lib/async/http/protocol/http.rb @@ -2,6 +2,7 @@ # Released under the MIT License. # Copyright, 2024, by Thomas Morgan. +# Copyright, 2024, by Samuel Williams. require_relative 'http1' require_relative 'http2' diff --git a/license.md b/license.md index 598f6a1..daa72d1 100644 --- a/license.md +++ b/license.md @@ -7,7 +7,7 @@ Copyright, 2019, by Denis Talakevich. Copyright, 2019-2020, by Brian Morearty. Copyright, 2019, by Cyril Roelandt. Copyright, 2020, by Stefan Wrobel. -Copyright, 2020, by Igor Sidorov. +Copyright, 2020-2024, by Igor Sidorov. Copyright, 2020, by Bruno Sutic. Copyright, 2020, by Sam Shadwell. Copyright, 2020, by Orgad Shaneh. @@ -21,6 +21,7 @@ Copyright, 2023-2024, by Thomas Morgan. Copyright, 2023, by dependabot[bot]. Copyright, 2023, by Josh Huber. Copyright, 2024, by Anton Zhuravsky. +Copyright, 2024, by Hal Brodigan. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/test/async/http/internet.rb b/test/async/http/internet.rb index 0eaf42a..43d7aa4 100644 --- a/test/async/http/internet.rb +++ b/test/async/http/internet.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2024, by Igor Sidorov. +# Copyright, 2024, by Hal Brodigan. require 'async/http/internet' require 'async/reactor' diff --git a/test/async/http/protocol/http.rb b/test/async/http/protocol/http.rb index d787dc9..90a2b5a 100755 --- a/test/async/http/protocol/http.rb +++ b/test/async/http/protocol/http.rb @@ -2,6 +2,7 @@ # Released under the MIT License. # Copyright, 2024, by Thomas Morgan. +# Copyright, 2024, by Samuel Williams. require 'async/http/protocol/http' require 'async/http/a_protocol' From e1e1f13ad040781cbe8afca0cddc80f531e72154 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 23 Jun 2024 23:43:09 +0900 Subject: [PATCH 026/125] Support for mocked client requests. (#41) --- guides/links.yaml | 2 + guides/mocking/readme.md | 75 +++++++++++++++++++++++++++++++++ lib/async/http.rb | 2 + lib/async/http/mock.rb | 23 ++++++++++ lib/async/http/mock/endpoint.rb | 70 ++++++++++++++++++++++++++++++ test/async/http/mock.rb | 74 ++++++++++++++++++++++++++++++++ 6 files changed, 246 insertions(+) create mode 100644 guides/links.yaml create mode 100644 guides/mocking/readme.md create mode 100644 lib/async/http/mock.rb create mode 100644 lib/async/http/mock/endpoint.rb create mode 100644 test/async/http/mock.rb diff --git a/guides/links.yaml b/guides/links.yaml new file mode 100644 index 0000000..d1b7ca5 --- /dev/null +++ b/guides/links.yaml @@ -0,0 +1,2 @@ +mocking: + order: 5 diff --git a/guides/mocking/readme.md b/guides/mocking/readme.md new file mode 100644 index 0000000..124d271 --- /dev/null +++ b/guides/mocking/readme.md @@ -0,0 +1,75 @@ +# Mocking + +This guide explains how to modify `Async::HTTP::Client` for mocking responses in tests. + +## Mocking HTTP Responses + +The mocking feature of `Async::HTTP` uses a real server running in a separate task, and routes all requests to it. This allows you to intercept requests and return custom responses, but still use the real HTTP client. + +In order to enable this feature, you must create an instance of {ruby Async::HTTP::Mock::Endpoint} which will handle the requests. + +~~~ ruby +require 'async/http' +require 'async/http/mock' + +mock_endpoint = Async::HTTP::Mock::Endpoint.new + +Sync do + # Start a background server: + server_task = Async(transient: true) do + mock_endpoint.run do |request| + # Respond to the request: + ::Protocol::HTTP::Response[200, {}, ["Hello, World"]] + end + end + + endpoint = Async::HTTP::Endpoint.parse("https://www.google.com") + mocked_endpoint = mock_endpoint.wrap(endpoint) + client = Async::HTTP::Client.new(mocked_endpoint) + + response = client.get("/") + puts response.read + # => "Hello, World" +end +~~~ + +## Transparent Mocking + +Using your test framework's mocking capabilities, you can easily replace the `Async::HTTP::Client#new` with a method that returns a client with a mocked endpoint. + +### Sus Integration + +~~~ ruby +require 'async/http' +require 'async/http/mock' +require 'sus/fixtures/async/reactor_context' + +include Sus::Fixtures::Async::ReactorContext + +let(:mock_endpoint) {Async::HTTP::Mock::Endpoint.new} + +def before + super + + # Mock the HTTP client: + mock(Async::HTTP::Client) do |mock| + mock.wrap(:new) do |original, endpoint| + original.call(mock_endpoint.wrap(endpoint)) + end + end + + # Run the mock server: + Async(transient: true) do + mock_endpoint.run do |request| + ::Protocol::HTTP::Response[200, {}, ["Hello, World"]] + end + end +end + +it "should perform a web request" do + client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://www.google.com")) + response = client.get("/") + # The response is mocked: + expect(response.read).to be == "Hello, World" +end +~~~ diff --git a/lib/async/http.rb b/lib/async/http.rb index 6b67b2b..3eac9b4 100644 --- a/lib/async/http.rb +++ b/lib/async/http.rb @@ -8,4 +8,6 @@ require_relative 'http/client' require_relative 'http/server' +require_relative 'http/internet' + require_relative 'http/endpoint' diff --git a/lib/async/http/mock.rb b/lib/async/http/mock.rb new file mode 100644 index 0000000..050f1c2 --- /dev/null +++ b/lib/async/http/mock.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +# +# Copyright, 2019, by Samuel G. D. Williams. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +require_relative 'mock/endpoint' diff --git a/lib/async/http/mock/endpoint.rb b/lib/async/http/mock/endpoint.rb new file mode 100644 index 0000000..798caef --- /dev/null +++ b/lib/async/http/mock/endpoint.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true +# +# Copyright, 2019, by Samuel G. D. Williams. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +require_relative '../protocol' + +require 'async/queue' + +module Async + module HTTP + module Mock + # This is an endpoint which bridges a client with a local server. + class Endpoint + def initialize(protocol = Protocol::HTTP2, scheme = "http", authority = "localhost", queue: Queue.new) + @protocol = protocol + @scheme = scheme + @authority = authority + + @queue = queue + end + + attr :protocol + attr :scheme + attr :authority + + # Processing incoming connections + # @yield [::HTTP::Protocol::Request] the requests as they come in. + def run(parent: Task.current, &block) + while peer = @queue.dequeue + server = @protocol.server(peer) + + parent.async do + server.each(&block) + end + end + end + + def connect + local, remote = ::Socket.pair(Socket::AF_UNIX, Socket::SOCK_STREAM) + + @queue.enqueue(remote) + + return local + end + + def wrap(endpoint) + self.class.new(@protocol, endpoint.scheme, endpoint.authority, queue: @queue) + end + end + end + end +end diff --git a/test/async/http/mock.rb b/test/async/http/mock.rb new file mode 100644 index 0000000..d54840c --- /dev/null +++ b/test/async/http/mock.rb @@ -0,0 +1,74 @@ +# Copyright, 2019, by Samuel G. D. Williams. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +require 'async/http/mock' +require 'async/http/endpoint' +require 'async/http/client' + +require 'sus/fixtures/async/reactor_context' + +describe Async::HTTP::Mock do + include Sus::Fixtures::Async::ReactorContext + + let(:endpoint) {Async::HTTP::Mock::Endpoint.new} + + it "can respond to requests" do + server = Async do + endpoint.run do |request| + ::Protocol::HTTP::Response[200, [], ["Hello World"]] + end + end + + client = Async::HTTP::Client.new(endpoint) + + response = client.get("/index") + + expect(response).to be(:success?) + expect(response.read).to be == "Hello World" + end + + with 'mocked client' do + it "can mock a client" do + server = Async do + endpoint.run do |request| + ::Protocol::HTTP::Response[200, [], ["Authority: #{request.authority}"]] + end + end + + mock(Async::HTTP::Client) do |mock| + replacement_endpoint = self.endpoint + + mock.wrap(:new) do |original, original_endpoint, **options| + original.call(replacement_endpoint.wrap(original_endpoint), **options) + end + end + + google_endpoint = Async::HTTP::Endpoint.parse("https://www.google.com") + client = Async::HTTP::Client.new(google_endpoint) + + response = client.get("/search?q=hello") + + expect(response).to be(:success?) + expect(response.read).to be == "Authority: www.google.com" + ensure + response&.close + end + end +end From fcf231a70e90f3e663c5dd81a2e20714861a9bdb Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 24 Jun 2024 10:13:49 +0900 Subject: [PATCH 027/125] Updated documentation + improved Internet interface. --- gems.rb | 3 - guides/getting-started/readme.md | 151 +++++++++++ guides/links.yaml | 6 +- guides/{mocking => testing}/readme.md | 6 +- lib/async/http/internet.rb | 16 +- lib/async/http/internet/instance.rb | 17 +- readme.md | 351 +------------------------- 7 files changed, 188 insertions(+), 362 deletions(-) create mode 100644 guides/getting-started/readme.md rename guides/{mocking => testing}/readme.md (89%) diff --git a/gems.rb b/gems.rb index dbbd8bd..26cdf5a 100644 --- a/gems.rb +++ b/gems.rb @@ -44,7 +44,4 @@ gem "localhost" gem "rack-test" - - # Optional dependency: - gem "thread-local" end diff --git a/guides/getting-started/readme.md b/guides/getting-started/readme.md new file mode 100644 index 0000000..7d56fba --- /dev/null +++ b/guides/getting-started/readme.md @@ -0,0 +1,151 @@ +# Getting Started + +This guide explains how to get started with `Async::HTTP`. + +## Installation + +Add the gem to your project: + +~~~ bash +$ bundle add async-http +~~~ + +## Core Concepts + +- {ruby Async::HTTP::Client} is the main class for making HTTP requests. +- {ruby Async::HTTP::Internet} provides a simple interface for making requests to any server "on the internet". +- {ruby Async::HTTP::Server} is the main class for handling HTTP requests. +- {ruby Async::HTTP::Endpoint} can parse HTTP URLs in order to create a client or server. +- [`protocol-http`](https://github.com/socketry/protocol-http) provides the abstract HTTP protocol interfaces. + +## Usage + +### Making a Request + +To make a request, create an instance of {ruby Async::HTTP::Internet} and call the appropriate method: + +~~~ ruby +require 'async/http/internet' + +Sync do + Async::HTTP::Internet.get("https://httpbin.org/get") do |response| + puts response.read + end +end +~~~ + +The following methods are supported: + +~~~ ruby +Async::HTTP::Internet.methods(false) +# => [:patch, :options, :connect, :post, :get, :delete, :head, :trace, :put] +~~~ + +Using a block will automatically close the response when the block completes. If you want to keep the response open, you can manage it manually: + +~~~ ruby +require 'async/http' + +Sync do + response = Async::HTTP::Internet.get("https://httpbin.org/get") + puts response.read +ensure + response&.close +end +~~~ + +As responses are streamed, you must ensure it is closed when you are finished with it. + +#### Persistence + +By default, {ruby Async::HTTP::Internet} will create a {ruby Async::HTTP::Client} for each remote host you communicate with, and will keep those connections open for as long as possible. This is useful for reducing the latency of subsequent requests to the same host. When you exit the event loop, the connections will be closed automatically. + +### Downloading a File + +~~~ ruby +require 'async/http' +require 'async/http/internet/instance' + +Sync do + # Issue a GET request to Google: + response = Async::HTTP::Internet.get("https://www.google.com/search?q=kittens") + + # Save the response body to a local file: + response.save("/tmp/search.html") +ensure + response&.close +end +~~~ + +### Posting Data + +To post data, use the `post` method: + +~~~ ruby +require 'async/http' +require 'async/http/internet/instance' + +data = {'life' => 42} + +Sync do + # Prepare the request: + headers = [['accept', 'application/json']] + body = JSON.dump(data) + + # Issues a POST request: + response = Async::HTTP::Internet.post("https://httpbin.org/anything", headers, body) + + # Save the response body to a local file: + pp JSON.parse(response.read) +ensure + response&.close +end +~~~ + +For more complex scenarios, including HTTP APIs, consider using [async-rest](https://github.com/socketry/async-rest) instead. + +### Timeouts + +To set a timeout for a request, use the `Task#with_timeout` method: + +~~~ ruby +require 'async/http' +require 'async/http/internet/instance' + +Sync do |task| + # Request will timeout after 2 seconds + task.with_timeout(2) do + response = Async::HTTP::Internet.get "https://httpbin.org/delay/10" + ensure + response&.close + end +rescue Async::TimeoutError + puts "The request timed out" +end +~~~ + +### Making a Server + +To create a server, use an instance of {ruby Async::HTTP::Server}: + +~~~ ruby +require 'async/http' + +endpoint = Async::HTTP::Endpoint.parse('http://localhost:9292') + +Sync do |task| + Async(transient: true) do + server = Async::HTTP::Server.for(endpoint) do |request| + ::Protocol::HTTP::Response[200, {}, ["Hello World"]] + end + + server.run + end + + client = Async::HTTP::Client.new(endpoint) + response = client.get("/") + puts response.read +ensure + response&.close +end +~~~ diff --git a/guides/links.yaml b/guides/links.yaml index d1b7ca5..89fed4c 100644 --- a/guides/links.yaml +++ b/guides/links.yaml @@ -1,2 +1,4 @@ -mocking: - order: 5 +getting-started: + order: 0 +testing: + order: 1 diff --git a/guides/mocking/readme.md b/guides/testing/readme.md similarity index 89% rename from guides/mocking/readme.md rename to guides/testing/readme.md index 124d271..a901454 100644 --- a/guides/mocking/readme.md +++ b/guides/testing/readme.md @@ -1,6 +1,8 @@ -# Mocking +# Testing -This guide explains how to modify `Async::HTTP::Client` for mocking responses in tests. +This guide explains how to use `Async::HTTP` clients and servers in your tests. + +In general, you should avoid making real HTTP requests in your tests. Instead, you should use a mock server or a fake client. ## Mocking HTTP Responses diff --git a/lib/async/http/internet.rb b/lib/async/http/internet.rb index 37fd379..37ee38f 100644 --- a/lib/async/http/internet.rb +++ b/lib/async/http/internet.rb @@ -39,7 +39,7 @@ def client_for(endpoint) # @parameter url [String] The URL to request, e.g. `https://www.codeotaku.com`. # @parameter headers [Hash | Protocol::HTTP::Headers] The headers to send with the request. # @parameter body [String | Protocol::HTTP::Body] The body to send with the request. - def call(method, url, headers = nil, body = nil) + def call(method, url, headers = nil, body = nil, &block) endpoint = Endpoint[url] client = self.client_for(endpoint) @@ -48,7 +48,15 @@ def call(method, url, headers = nil, body = nil) request = ::Protocol::HTTP::Request.new(endpoint.scheme, endpoint.authority, method, endpoint.path, nil, headers, body) - return client.call(request) + response = client.call(request) + + return response unless block_given? + + begin + yield response + ensure + response.close + end end def close @@ -60,8 +68,8 @@ def close end ::Protocol::HTTP::Methods.each do |name, verb| - define_method(verb.downcase) do |url, headers = nil, body = nil| - self.call(verb, url, headers, body) + define_method(verb.downcase) do |url, headers = nil, body = nil, &block| + self.call(verb, url, headers, body, &block) end end diff --git a/lib/async/http/internet/instance.rb b/lib/async/http/internet/instance.rb index 8581fed..ec16ee6 100644 --- a/lib/async/http/internet/instance.rb +++ b/lib/async/http/internet/instance.rb @@ -4,13 +4,24 @@ # Copyright, 2021-2023, by Samuel Williams. require_relative '../internet' -require 'thread/local' + +::Thread.attr_accessor :async_http_internet_instance module Async module HTTP class Internet - # Provide access to a shared thread-local instance. - extend ::Thread::Local + # The global instance of the internet. + def self.instance + ::Thread.current.async_http_internet_instance ||= self.new + end + + class << self + ::Protocol::HTTP::Methods.each do |name, verb| + define_method(verb.downcase) do |url, headers = nil, body = nil, &block| + self.instance.call(verb, url, headers, body, &block) + end + end + end end end end diff --git a/readme.md b/readme.md index 1a22fe6..79fc3e2 100644 --- a/readme.md +++ b/readme.md @@ -4,358 +4,13 @@ An asynchronous client and server implementation of HTTP/1.0, HTTP/1.1 and HTTP/ [![Development Status](https://github.com/socketry/async-http/workflows/Test/badge.svg)](https://github.com/socketry/async-http/actions?workflow=Test) -## Installation - -Add this line to your application's Gemfile: - -``` ruby -gem 'async-http' -``` - -And then execute: - - $ bundle - -Or install it yourself as: - - $ gem install async-http - ## Usage -Please see the [project documentation](https://socketry.github.io/async-http/) or serve it locally using `bake utopia:project:serve`. - -### Post JSON data - -Here is an example showing how to post a data structure as JSON to a remote resource: - -``` ruby -#!/usr/bin/env ruby - -require 'json' -require 'async' -require 'async/http/internet' - -data = {'life' => 42} - -Async do - # Make a new internet: - internet = Async::HTTP::Internet.new - - # Prepare the request: - headers = [['accept', 'application/json']] - body = [JSON.dump(data)] - - # Issues a POST request: - response = internet.post("https://httpbin.org/anything", headers, body) - - # Save the response body to a local file: - pp JSON.parse(response.read) -ensure - # The internet is closed for business: - internet.close -end -``` - -Consider using [async-rest](https://github.com/socketry/async-rest) instead. - -### Multiple Requests - -To issue multiple requests concurrently, you should use a barrier, e.g. - -``` ruby -#!/usr/bin/env ruby - -require 'async' -require 'async/barrier' -require 'async/http/internet' - -TOPICS = ["ruby", "python", "rust"] - -Async do - internet = Async::HTTP::Internet.new - barrier = Async::Barrier.new - - # Spawn an asynchronous task for each topic: - TOPICS.each do |topic| - barrier.async do - response = internet.get "https://www.google.com/search?q=#{topic}" - puts "Found #{topic}: #{response.read.scan(topic).size} times." - end - end - - # Ensure we wait for all requests to complete before continuing: - barrier.wait -ensure - internet&.close -end -``` - -#### Limiting Requests - -If you need to limit the number of simultaneous requests, use a semaphore. - -``` ruby -#!/usr/bin/env ruby - -require 'async' -require 'async/barrier' -require 'async/semaphore' -require 'async/http/internet' - -TOPICS = ["ruby", "python", "rust"] - -Async do - internet = Async::HTTP::Internet.new - barrier = Async::Barrier.new - semaphore = Async::Semaphore.new(2, parent: barrier) - - # Spawn an asynchronous task for each topic: - TOPICS.each do |topic| - semaphore.async do - response = internet.get "https://www.google.com/search?q=#{topic}" - puts "Found #{topic}: #{response.read.scan(topic).size} times." - end - end - - # Ensure we wait for all requests to complete before continuing: - barrier.wait -ensure - internet&.close -end -``` - -### Persistent Connections - -To keep connections alive, install the `thread-local` gem, -require `async/http/internet/instance`, and use the `instance`, e.g. - -``` ruby -#!/usr/bin/env ruby - -require 'async' -require 'async/http/internet/instance' - -Async do - internet = Async::HTTP::Internet.instance - response = internet.get "https://www.google.com/search?q=test" - puts "Found #{response.read.size} results." -end -``` - -### Downloading a File - -Here is an example showing how to download a file and save it to a local path: - -``` ruby -#!/usr/bin/env ruby - -require 'async' -require 'async/http/internet' - -Async do - # Make a new internet: - internet = Async::HTTP::Internet.new - - # Issues a GET request to Google: - response = internet.get("https://www.google.com/search?q=kittens") - - # Save the response body to a local file: - response.save("/tmp/search.html") -ensure - # The internet is closed for business: - internet.close -end -``` - -### Basic Client/Server - -Here is a basic example of a client/server running in the same reactor: - -``` ruby -#!/usr/bin/env ruby - -require 'async' -require 'async/http/server' -require 'async/http/client' -require 'async/http/endpoint' -require 'async/http/protocol/response' - -endpoint = Async::HTTP::Endpoint.parse('http://127.0.0.1:9294') - -app = lambda do |request| - Protocol::HTTP::Response[200, {}, ["Hello World"]] -end - -server = Async::HTTP::Server.new(app, endpoint) -client = Async::HTTP::Client.new(endpoint) - -Async do |task| - server_task = task.async do - server.run - end - - response = client.get("/") - - puts response.status - puts response.read - - server_task.stop -end -``` - -### Advanced Verification - -You can hook into SSL certificate verification to improve server verification. - -``` ruby -require 'async' -require 'async/http' - -# These are generated from the certificate chain that the server presented. -trusted_fingerprints = { - "dac9024f54d8f6df94935fb1732638ca6ad77c13" => true, - "e6a3b45b062d509b3382282d196efe97d5956ccb" => true, - "07d63f4c05a03f1c306f9941b8ebf57598719ea2" => true, - "e8d994f44ff20dc78dbff4e59d7da93900572bbf" => true, -} - -Async do - endpoint = Async::HTTP::Endpoint.parse("https://www.codeotaku.com/index") - - # This is a quick hack/POC: - ssl_context = endpoint.ssl_context - - ssl_context.verify_callback = proc do |verified, store_context| - certificate = store_context.current_cert - fingerprint = OpenSSL::Digest::SHA1.new(certificate.to_der).to_s - - if trusted_fingerprints.include? fingerprint - true - else - Console.logger.warn("Untrusted Certificate Fingerprint"){fingerprint} - false - end - end - - endpoint = endpoint.with(ssl_context: ssl_context) - - client = Async::HTTP::Client.new(endpoint) - - response = client.get(endpoint.path) - - pp response.status, response.headers.fields, response.read -end -``` - -### Timeouts - -Here's a basic example with a timeout: - -``` ruby -#!/usr/bin/env ruby - -require 'async/http/internet' - -Async do |task| - internet = Async::HTTP::Internet.new - - # Request will timeout after 2 seconds - task.with_timeout(2) do - response = internet.get "https://httpbin.org/delay/10" - end -rescue Async::TimeoutError - puts "The request timed out" -ensure - internet&.close -end -``` - -## Performance - -On a 4-core 8-thread i7, running `ab` which uses discrete (non-keep-alive) connections: - - $ ab -c 8 -t 10 http://127.0.0.1:9294/ - This is ApacheBench, Version 2.3 <$Revision: 1757674 $> - Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ - Licensed to The Apache Software Foundation, http://www.apache.org/ - - Benchmarking 127.0.0.1 (be patient) - Completed 5000 requests - Completed 10000 requests - Completed 15000 requests - Completed 20000 requests - Completed 25000 requests - Completed 30000 requests - Completed 35000 requests - Completed 40000 requests - Completed 45000 requests - Completed 50000 requests - Finished 50000 requests - - - Server Software: - Server Hostname: 127.0.0.1 - Server Port: 9294 - - Document Path: / - Document Length: 13 bytes - - Concurrency Level: 8 - Time taken for tests: 1.869 seconds - Complete requests: 50000 - Failed requests: 0 - Total transferred: 2450000 bytes - HTML transferred: 650000 bytes - Requests per second: 26755.55 [#/sec] (mean) - Time per request: 0.299 [ms] (mean) - Time per request: 0.037 [ms] (mean, across all concurrent requests) - Transfer rate: 1280.29 [Kbytes/sec] received - - Connection Times (ms) - min mean[+/-sd] median max - Connect: 0 0 0.0 0 0 - Processing: 0 0 0.2 0 6 - Waiting: 0 0 0.2 0 6 - Total: 0 0 0.2 0 6 - - Percentage of the requests served within a certain time (ms) - 50% 0 - 66% 0 - 75% 0 - 80% 0 - 90% 0 - 95% 1 - 98% 1 - 99% 1 - 100% 6 (longest request) - -On a 4-core 8-thread i7, running `wrk`, which uses 8 keep-alive connections: - - $ wrk -c 8 -d 10 -t 8 http://127.0.0.1:9294/ - Running 10s test @ http://127.0.0.1:9294/ - 8 threads and 8 connections - Thread Stats Avg Stdev Max +/- Stdev - Latency 217.69us 0.99ms 23.21ms 97.39% - Req/Sec 12.18k 1.58k 17.67k 83.21% - 974480 requests in 10.10s, 60.41MB read - Requests/sec: 96485.00 - Transfer/sec: 5.98MB - -According to these results, the cost of handling connections is quite high, while general throughput seems pretty decent. - -## Semantic Model - -### Scheme - -HTTP/1 has an implicit scheme determined by the kind of connection made to the server (either `http` or `https`), while HTTP/2 models this explicitly and the client indicates this in the request using the `:scheme` pseudo-header (typically `https`). To normalize this, `Async::HTTP::Client` and `Async::HTTP::Server` have a default scheme which is used if none is supplied. - -### Version - -HTTP/1 has an explicit version while HTTP/2 does not expose the version in any way. +Please see the [project documentation](https://socketry.github.io/async-http/) for more details. -### Reason + - [Getting Started](https://socketry.github.io/async-http/guides/getting-started/index) - This guide explains how to get started with `Async::HTTP`. -HTTP/1 responses contain a reason field which is largely irrelevant. HTTP/2 does not support this field. + - [Testing](https://socketry.github.io/async-http/guides/testing/index) - This guide explains how to use `Async::HTTP` clients and servers in your tests. ## Contributing From 6a10cc4b4725bdb4bdeb1021d490bad1e1543ecc Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 24 Jun 2024 10:18:07 +0900 Subject: [PATCH 028/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 062ec90..5f22c06 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.68.0" + VERSION = "0.69.0" end end From ad0caae2f6041dad40fb3c3a3ec6201e71a06b55 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 24 Jun 2024 10:20:26 +0900 Subject: [PATCH 029/125] Minor improvement to guide. --- guides/getting-started/readme.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/guides/getting-started/readme.md b/guides/getting-started/readme.md index 7d56fba..3f67483 100644 --- a/guides/getting-started/readme.md +++ b/guides/getting-started/readme.md @@ -22,10 +22,10 @@ $ bundle add async-http ### Making a Request -To make a request, create an instance of {ruby Async::HTTP::Internet} and call the appropriate method: +To make a request, use {ruby Async::HTTP::Internet} and call the appropriate method: ~~~ ruby -require 'async/http/internet' +require 'async/http/internet/instance' Sync do Async::HTTP::Internet.get("https://httpbin.org/get") do |response| @@ -44,7 +44,7 @@ Async::HTTP::Internet.methods(false) Using a block will automatically close the response when the block completes. If you want to keep the response open, you can manage it manually: ~~~ ruby -require 'async/http' +require 'async/http/internet/instance' Sync do response = Async::HTTP::Internet.get("https://httpbin.org/get") @@ -63,7 +63,6 @@ By default, {ruby Async::HTTP::Internet} will create a {ruby Async::HTTP::Client ### Downloading a File ~~~ ruby -require 'async/http' require 'async/http/internet/instance' Sync do @@ -82,7 +81,6 @@ end To post data, use the `post` method: ~~~ ruby -require 'async/http' require 'async/http/internet/instance' data = {'life' => 42} @@ -109,7 +107,6 @@ For more complex scenarios, including HTTP APIs, consider using [async-rest](htt To set a timeout for a request, use the `Task#with_timeout` method: ~~~ ruby -require 'async/http' require 'async/http/internet/instance' Sync do |task| From 004dff5e82af2d35403df7ef1981f0da2420616b Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 12 Jul 2024 19:40:17 +1200 Subject: [PATCH 030/125] Fix return value type. --- lib/async/http/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/server.rb b/lib/async/http/server.rb index 0d4790c..311bd61 100755 --- a/lib/async/http/server.rb +++ b/lib/async/http/server.rb @@ -64,7 +64,7 @@ def accept(peer, address, task: Task.current) connection&.close end - # @returns [Array(Async::Task)] The task that is running the server. + # @returns [Async::Task] The task that is running the server. def run Async do |task| @endpoint.accept(&self.method(:accept)) From 7c4ed31247d0b2e131e1bb411dd4c63ad45a638b Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 13 Jul 2024 15:47:16 +1200 Subject: [PATCH 031/125] Better handling of supported schemes. (#166) --- lib/async/http/endpoint.rb | 12 +++++++++++- test/async/http/endpoint.rb | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index dd80f9c..3bd1e23 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -18,6 +18,10 @@ module Async module HTTP # Represents a way to connect to a remote HTTP server. class Endpoint < ::IO::Endpoint::Generic + SCHEMES = ['HTTP', 'HTTPS', 'WS', 'WSS'].to_h do |scheme| + [scheme.downcase, URI.scheme_list[scheme]] + end + def self.parse(string, endpoint = nil, **options) url = URI.parse(string).normalize @@ -25,9 +29,15 @@ def self.parse(string, endpoint = nil, **options) end # Construct an endpoint with a specified scheme, hostname, optional path, and options. + # + # @parameter scheme [String] The scheme to use, e.g. "http" or "https". + # @parameter hostname [String] The hostname to connect to (or bind to). + # @parameter *options [Hash] Additional options, passed to {#initialize}. def self.for(scheme, hostname, path = "/", **options) # TODO: Consider using URI.for once it becomes available: - uri_klass = URI.scheme_list[scheme.upcase] || URI::HTTP + uri_klass = SCHEMES.fetch(scheme.downcase) do + raise ArgumentError, "Unsupported scheme: #{scheme}" + end self.new( uri_klass.new(scheme, nil, hostname, nil, nil, path, nil, nil, nil).normalize, diff --git a/test/async/http/endpoint.rb b/test/async/http/endpoint.rb index 9c501bd..f2e82ba 100644 --- a/test/async/http/endpoint.rb +++ b/test/async/http/endpoint.rb @@ -81,6 +81,18 @@ expect(subject).not.to be(:secure?) end end + + with 'invalid scheme' do + it "should raise an argument error" do + expect do + Async::HTTP::Endpoint.for("foo", "localhost") + end.to raise_exception(ArgumentError, message: be =~ /scheme/) + + expect do + Async::HTTP::Endpoint.for(:http, "localhost", "/foo") + end.to raise_exception(ArgumentError, message: be =~ /scheme/) + end + end end with '#secure?' do From cac897311ff5636a257d45351442ada4842f0279 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 13 Jul 2024 16:01:45 +1200 Subject: [PATCH 032/125] Improvements to scheme list and error messages. --- lib/async/http/endpoint.rb | 11 +++++++---- test/async/http/endpoint.rb | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index 3bd1e23..9006a3e 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -18,9 +18,12 @@ module Async module HTTP # Represents a way to connect to a remote HTTP server. class Endpoint < ::IO::Endpoint::Generic - SCHEMES = ['HTTP', 'HTTPS', 'WS', 'WSS'].to_h do |scheme| - [scheme.downcase, URI.scheme_list[scheme]] - end + SCHEMES = { + 'http' => URI::HTTP, + 'https' => URI::HTTPS, + 'ws' => URI::WS, + 'wss' => URI::WSS, + } def self.parse(string, endpoint = nil, **options) url = URI.parse(string).normalize @@ -36,7 +39,7 @@ def self.parse(string, endpoint = nil, **options) def self.for(scheme, hostname, path = "/", **options) # TODO: Consider using URI.for once it becomes available: uri_klass = SCHEMES.fetch(scheme.downcase) do - raise ArgumentError, "Unsupported scheme: #{scheme}" + raise ArgumentError, "Unsupported scheme: #{scheme.inspect}" end self.new( diff --git a/test/async/http/endpoint.rb b/test/async/http/endpoint.rb index f2e82ba..e98e7c7 100644 --- a/test/async/http/endpoint.rb +++ b/test/async/http/endpoint.rb @@ -86,11 +86,11 @@ it "should raise an argument error" do expect do Async::HTTP::Endpoint.for("foo", "localhost") - end.to raise_exception(ArgumentError, message: be =~ /scheme/) + end.to raise_exception(ArgumentError, message: be =~ /Unsupported scheme: "foo"/) expect do Async::HTTP::Endpoint.for(:http, "localhost", "/foo") - end.to raise_exception(ArgumentError, message: be =~ /scheme/) + end.to raise_exception(ArgumentError, message: be =~ /Unsupported scheme: :http/) end end end From 8bbf377044cb1660c50f315428e3df70129cf694 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 15 Jul 2024 13:16:05 +1200 Subject: [PATCH 033/125] Remove personal website from tests. --- test/async/http/client/codeotaku.rb | 42 ----------------------------- test/async/http/client/google.rb | 18 +++++++++++++ test/async/http/internet.rb | 4 +-- 3 files changed, 20 insertions(+), 44 deletions(-) delete mode 100644 test/async/http/client/codeotaku.rb diff --git a/test/async/http/client/codeotaku.rb b/test/async/http/client/codeotaku.rb deleted file mode 100644 index 5dfe748..0000000 --- a/test/async/http/client/codeotaku.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -# Released under the MIT License. -# Copyright, 2023, by Samuel Williams. - -require 'async/http/client' -require 'async/http/endpoint' -require 'protocol/http/accept_encoding' - -require 'sus/fixtures/async' - -describe Async::HTTP::Client do - include Sus::Fixtures::Async::ReactorContext - - let(:endpoint) {Async::HTTP::Endpoint.parse('https://www.codeotaku.com')} - let(:client) {Async::HTTP::Client.new(endpoint)} - - it "should specify hostname" do - expect(endpoint.hostname).to be == "www.codeotaku.com" - expect(client.authority).to be == "www.codeotaku.com" - end - - it 'can fetch remote resource' do - response = client.get('/index') - - response.finish - - expect(response).not.to be(:failure?) - end - - it "can request remote resource with compression" do - compressor = Protocol::HTTP::AcceptEncoding.new(client) - - response = compressor.get("/index", {'accept-encoding' => 'gzip'}) - - expect(response).to be(:success?) - - expect(response.body).to be_a Async::HTTP::Body::Inflate - expect(response.read).to be(:start_with?, '') - end -end - diff --git a/test/async/http/client/google.rb b/test/async/http/client/google.rb index f8c8ae2..c1232d7 100644 --- a/test/async/http/client/google.rb +++ b/test/async/http/client/google.rb @@ -6,6 +6,8 @@ require 'async/http/client' require 'async/http/endpoint' +require 'protocol/http/accept_encoding' + require 'sus/fixtures/async' describe Async::HTTP::Client do @@ -14,6 +16,11 @@ let(:endpoint) {Async::HTTP::Endpoint.parse('https://www.google.com')} let(:client) {Async::HTTP::Client.new(endpoint)} + it "should specify a hostname" do + expect(endpoint.hostname).to be == "www.google.com" + expect(client.authority).to be == "www.google.com" + end + it 'can fetch remote resource' do response = client.get('/', 'accept' => '*/*') @@ -23,4 +30,15 @@ client.close end + + it "can request remote resource with compression" do + compressor = Protocol::HTTP::AcceptEncoding.new(client) + + response = compressor.get("/", {'accept-encoding' => 'gzip'}) + + expect(response).to be(:success?) + + expect(response.body).to be_a Async::HTTP::Body::Inflate + expect(response.read).to be(:start_with?, '') + end end diff --git a/test/async/http/internet.rb b/test/async/http/internet.rb index 43d7aa4..d8a9550 100644 --- a/test/async/http/internet.rb +++ b/test/async/http/internet.rb @@ -18,7 +18,7 @@ let(:headers) {[['accept', '*/*'], ['user-agent', 'async-http']]} it "can fetch remote website" do - response = internet.get("https://www.codeotaku.com/index", headers) + response = internet.get("https://www.google.com/", headers) expect(response).to be(:success?) @@ -26,7 +26,7 @@ end it "can accept URI::HTTP objects" do - uri = URI.parse("https://www.codeotaku.com/index") + uri = URI.parse("https://www.google.com/") response = internet.get(uri, headers) expect(response).to be(:success?) From 3f7b32f5b6e53144e6dd7e4f88de1f5fe90d2fcc Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 28 Jul 2024 16:51:35 +1200 Subject: [PATCH 034/125] Use updated syntax. --- fixtures/async/http/a_protocol.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fixtures/async/http/a_protocol.rb b/fixtures/async/http/a_protocol.rb index 82e0e1f..6d7654f 100644 --- a/fixtures/async/http/a_protocol.rb +++ b/fixtures/async/http/a_protocol.rb @@ -247,9 +247,8 @@ module HTTP with 'with response' do let(:response) {client.get("/")} - def after + after do response.finish - super end it "can finish gracefully" do @@ -299,9 +298,8 @@ def after with 'POST' do let(:response) {client.post("/", {}, ["Hello", " ", "World"])} - def after + after do response.finish - super end it "is successful" do From 498b9126480dc4342b05a4e674a621ebed3ee65b Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 2 Aug 2024 17:18:00 +1200 Subject: [PATCH 035/125] Response protocol should only be a single value. (#162) --- lib/async/http/protocol/http1/response.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/async/http/protocol/http1/response.rb b/lib/async/http/protocol/http1/response.rb index 6027e19..7f30464 100644 --- a/lib/async/http/protocol/http1/response.rb +++ b/lib/async/http/protocol/http1/response.rb @@ -31,7 +31,8 @@ def initialize(connection, version, status, reason, headers, body) @connection = connection @reason = reason - protocol = headers.delete(UPGRADE) + # Technically, there should never be more than one value for the upgrade header, but we'll just take the first one to avoid complexity. + protocol = headers.delete(UPGRADE)&.first super(version, status, headers, body, protocol) end From ae738727d08c4e7cf26ac27fafe56dc0e816e023 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 14 Aug 2024 13:24:18 +1200 Subject: [PATCH 036/125] Modernize gem. (#173) --- .github/workflows/documentation-coverage.yaml | 25 ++++++++++ .github/workflows/rubocop.yaml | 24 ++++++++++ .../{coverage.yaml => test-coverage.yaml} | 4 +- .rubocop.yml | 46 +++++++++++++++++++ async-http.gemspec | 2 +- examples/fetch/config.ru | 1 + examples/trenni/Gemfile | 1 + gems.rb | 11 +++-- lib/async/http.rb | 2 +- lib/async/http/internet/instance.rb | 2 +- lib/async/http/mock.rb | 23 ++-------- lib/async/http/mock/endpoint.rb | 23 ++-------- readme.md | 16 +++---- test/async/http/body/pipe.rb | 2 +- test/async/http/client/google.rb | 2 +- test/async/http/mock.rb | 23 ++-------- 16 files changed, 128 insertions(+), 79 deletions(-) create mode 100644 .github/workflows/documentation-coverage.yaml create mode 100644 .github/workflows/rubocop.yaml rename .github/workflows/{coverage.yaml => test-coverage.yaml} (97%) create mode 100644 .rubocop.yml diff --git a/.github/workflows/documentation-coverage.yaml b/.github/workflows/documentation-coverage.yaml new file mode 100644 index 0000000..b3bac9a --- /dev/null +++ b/.github/workflows/documentation-coverage.yaml @@ -0,0 +1,25 @@ +name: Documentation Coverage + +on: [push, pull_request] + +permissions: + contents: read + +env: + CONSOLE_OUTPUT: XTerm + COVERAGE: PartialSummary + +jobs: + validate: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.3" + bundler-cache: true + + - name: Validate coverage + timeout-minutes: 5 + run: bundle exec bake decode:index:coverage lib diff --git a/.github/workflows/rubocop.yaml b/.github/workflows/rubocop.yaml new file mode 100644 index 0000000..287c06d --- /dev/null +++ b/.github/workflows/rubocop.yaml @@ -0,0 +1,24 @@ +name: RuboCop + +on: [push, pull_request] + +permissions: + contents: read + +env: + CONSOLE_OUTPUT: XTerm + +jobs: + check: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ruby + bundler-cache: true + + - name: Run RuboCop + timeout-minutes: 10 + run: bundle exec rubocop diff --git a/.github/workflows/coverage.yaml b/.github/workflows/test-coverage.yaml similarity index 97% rename from .github/workflows/coverage.yaml rename to .github/workflows/test-coverage.yaml index 68adbf2..f9da2ff 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -1,4 +1,4 @@ -name: Coverage +name: Test Coverage on: [push, pull_request] @@ -33,7 +33,7 @@ jobs: - name: Run tests timeout-minutes: 5 run: bundle exec bake test - + - uses: actions/upload-artifact@v3 with: name: coverage-${{matrix.os}}-${{matrix.ruby}} diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..442c667 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,46 @@ +AllCops: + DisabledByDefault: true + +Layout/IndentationStyle: + Enabled: true + EnforcedStyle: tabs + +Layout/InitialIndentation: + Enabled: true + +Layout/IndentationWidth: + Enabled: true + Width: 1 + +Layout/IndentationConsistency: + Enabled: true + EnforcedStyle: normal + +Layout/EndAlignment: + Enabled: true + EnforcedStyleAlignWith: start_of_line + +Layout/BeginEndAlignment: + Enabled: true + EnforcedStyleAlignWith: start_of_line + +Layout/ElseAlignment: + Enabled: true + +Layout/DefEndAlignment: + Enabled: true + +Layout/CaseIndentation: + Enabled: true + +Layout/CommentIndentation: + Enabled: true + +Layout/EmptyLinesAroundClassBody: + Enabled: true + +Layout/EmptyLinesAroundModuleBody: + Enabled: true + +Style/FrozenStringLiteralComment: + Enabled: true diff --git a/async-http.gemspec b/async-http.gemspec index f34cd30..ec72274 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.7" spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" - spec.add_dependency "protocol-http", "~> 0.26" + spec.add_dependency "protocol-http", "~> 0.28" spec.add_dependency "protocol-http1", "~> 0.19" spec.add_dependency "protocol-http2", "~> 0.18" spec.add_dependency "traces", ">= 0.10" diff --git a/examples/fetch/config.ru b/examples/fetch/config.ru index 1363f15..0a29fa6 100644 --- a/examples/fetch/config.ru +++ b/examples/fetch/config.ru @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'rack' diff --git a/examples/trenni/Gemfile b/examples/trenni/Gemfile index 507ac5e..d7bdde9 100644 --- a/examples/trenni/Gemfile +++ b/examples/trenni/Gemfile @@ -1,3 +1,4 @@ +# frozen_string_literal: true source 'https://rubygems.org' diff --git a/gems.rb b/gems.rb index 26cdf5a..1fe0e80 100644 --- a/gems.rb +++ b/gems.rb @@ -24,24 +24,25 @@ gem "bake-modernize" gem "bake-gem" - gem "falcon", "~> 0.46" + gem "falcon", "~> 0.47" gem "utopia-project" end group :test do - gem "covered" gem "sus" + gem "covered" + gem "decode" + gem "rubocop" + gem "sus-fixtures-async" gem "sus-fixtures-async-http", "~> 0.8" gem "sus-fixtures-openssl" - gem "bake" gem "bake-test" gem "bake-test-external" gem "async-container", "~> 0.14" - gem "async-rspec", "~> 1.10" - + gem "localhost" gem "rack-test" end diff --git a/lib/async/http.rb b/lib/async/http.rb index 3eac9b4..c18c359 100644 --- a/lib/async/http.rb +++ b/lib/async/http.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2023, by Samuel Williams. +# Copyright, 2017-2024, by Samuel Williams. require_relative 'http/version' diff --git a/lib/async/http/internet/instance.rb b/lib/async/http/internet/instance.rb index ec16ee6..1ea5832 100644 --- a/lib/async/http/internet/instance.rb +++ b/lib/async/http/internet/instance.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2021-2023, by Samuel Williams. +# Copyright, 2021-2024, by Samuel Williams. require_relative '../internet' diff --git a/lib/async/http/mock.rb b/lib/async/http/mock.rb index 050f1c2..d9ecae3 100644 --- a/lib/async/http/mock.rb +++ b/lib/async/http/mock.rb @@ -1,23 +1,6 @@ # frozen_string_literal: true -# -# Copyright, 2019, by Samuel G. D. Williams. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. + +# Released under the MIT License. +# Copyright, 2024, by Samuel Williams. require_relative 'mock/endpoint' diff --git a/lib/async/http/mock/endpoint.rb b/lib/async/http/mock/endpoint.rb index 798caef..91c9279 100644 --- a/lib/async/http/mock/endpoint.rb +++ b/lib/async/http/mock/endpoint.rb @@ -1,24 +1,7 @@ # frozen_string_literal: true -# -# Copyright, 2019, by Samuel G. D. Williams. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. + +# Released under the MIT License. +# Copyright, 2024, by Samuel Williams. require_relative '../protocol' diff --git a/readme.md b/readme.md index 79fc3e2..076e223 100644 --- a/readme.md +++ b/readme.md @@ -22,14 +22,6 @@ We welcome contributions to this project. 4. Push to the branch (`git push origin my-new-feature`). 5. Create new Pull Request. -### Developer Certificate of Origin - -This project uses the [Developer Certificate of Origin](https://developercertificate.org/). All contributors to this project must agree to this document to have their contributions accepted. - -### Contributor Covenant - -This project is governed by the [Contributor Covenant](https://www.contributor-covenant.org/). All contributors and participants agree to abide by its terms. - ## See Also - [benchmark-http](https://github.com/socketry/benchmark-http) — A benchmarking tool to report on web server concurrency. @@ -37,3 +29,11 @@ This project is governed by the [Contributor Covenant](https://www.contributor-c - [async-websocket](https://github.com/socketry/async-websocket) — Asynchronous client and server websockets. - [async-rest](https://github.com/socketry/async-rest) — A RESTful resource layer built on top of `async-http`. - [async-http-faraday](https://github.com/socketry/async-http-faraday) — A faraday adapter to use `async-http`. + +### Developer Certificate of Origin + +In order to protect users of this project, we require all contributors to comply with the [Developer Certificate of Origin](https://developercertificate.org/). This ensures that all contributions are properly licensed and attributed. + +### Community Guidelines + +This project is best served by a collaborative and respectful environment. Treat each other professionally, respect differing viewpoints, and engage constructively. Harassment, discrimination, or harmful behavior is not tolerated. Communicate clearly, listen actively, and support one another. If any issues arise, please inform the project maintainers. diff --git a/test/async/http/body/pipe.rb b/test/async/http/body/pipe.rb index 94cdcb0..0760de2 100644 --- a/test/async/http/body/pipe.rb +++ b/test/async/http/body/pipe.rb @@ -25,7 +25,7 @@ def before super - # input writer task + # input writer task Async do |task| first, second = data.split(' ') input.write("#{first} ") diff --git a/test/async/http/client/google.rb b/test/async/http/client/google.rb index c1232d7..cc1f70f 100644 --- a/test/async/http/client/google.rb +++ b/test/async/http/client/google.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. require 'async/http/client' require 'async/http/endpoint' diff --git a/test/async/http/mock.rb b/test/async/http/mock.rb index d54840c..8396fe1 100644 --- a/test/async/http/mock.rb +++ b/test/async/http/mock.rb @@ -1,22 +1,7 @@ -# Copyright, 2019, by Samuel G. D. Williams. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2024, by Samuel Williams. require 'async/http/mock' require 'async/http/endpoint' From 11b9d5d26a6afd7852d3f560b38f95d000d45c82 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 14 Aug 2024 13:36:58 +1200 Subject: [PATCH 037/125] Extract redirect location handling and clarify behaviour. (#172) --- lib/async/http/relative_location.rb | 47 +++++++++++++++++++++------- test/async/http/relative_location.rb | 22 +++++++++++-- 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/lib/async/http/relative_location.rb b/lib/async/http/relative_location.rb index ad71194..16f969f 100644 --- a/lib/async/http/relative_location.rb +++ b/lib/async/http/relative_location.rb @@ -16,7 +16,9 @@ module HTTP class TooManyRedirects < StandardError end - # A client wrapper which transparently handles both relative and absolute redirects to a given maximum number of hops. + # A client wrapper which transparently handles redirects to a given maximum number of hops. + # + # The default implementation will only follow relative locations (i.e. those without a scheme) and will switch to GET if the original request was not a GET. # # The best reference for these semantics is defined by the [Fetch specification](https://fetch.spec.whatwg.org/#http-redirect-fetch). # @@ -58,14 +60,38 @@ def redirect_with_get?(request, response) end end + # Handle a redirect to a relative location. + # + # @parameter request [Protocol::HTTP::Request] The original request, which you can modify if you want to handle the redirect. + # @parameter location [String] The relative location to redirect to. + # @returns [Boolean] True if the redirect was handled, false if it was not. + def handle_redirect(request, location) + uri = URI.parse(location) + + if uri.absolute? + return false + end + + # Update the path of the request: + request.path = Reference[request.path] + location + + # Follow the redirect: + return true + end + def call(request) # We don't want to follow redirects for HEAD requests: return super if request.head? if body = request.body - # We need to cache the body as it might be submitted multiple times if we get a response status of 307 or 308: - body = ::Protocol::HTTP::Body::Rewindable.new(body) - request.body = body + if body.respond_to?(:rewind) + # The request body was already rewindable, so use it as is: + body = request.body + else + # The request body was not rewindable, and we might need to resubmit it if we get a response status of 307 or 308, so make it rewindable: + body = ::Protocol::HTTP::Body::Rewindable.new(body) + request.body = body + end end hops = 0 @@ -83,23 +109,22 @@ def call(request) response.finish - uri = URI.parse(location) - - if uri.absolute? + unless handle_redirect(request, location) return response - else - request.path = Reference[request.path] + location end + # Ensure the request (body) is finished and set to nil before we manipulate the request: + request.finish + if request.method == GET or response.preserve_method? # We (might) need to rewind the body so that it can be submitted again: body&.rewind + request.body = body else # We are changing the method to GET: request.method = GET - # Clear the request body: - request.finish + # We will no longer be submitting the body: body = nil # Remove any headers which are not allowed in a GET request: diff --git a/test/async/http/relative_location.rb b/test/async/http/relative_location.rb index 4f453db..6d8c135 100644 --- a/test/async/http/relative_location.rb +++ b/test/async/http/relative_location.rb @@ -30,7 +30,10 @@ end it 'should redirect POST to GET' do - response = relative_location.post('/') + body = Protocol::HTTP::Body::Buffered.wrap(["Hello, World!"]) + expect(body).to receive(:finish) + + response = relative_location.post('/', {}, body) expect(response).to be(:success?) expect(response.read).to be == "GET" @@ -44,9 +47,22 @@ end it 'should fail with maximum redirects' do - expect{ + expect do response = relative_location.get('/home') - }.to raise_exception(Async::HTTP::TooManyRedirects, message: be =~ /maximum/) + end.to raise_exception(Async::HTTP::TooManyRedirects, message: be =~ /maximum/) + end + end + + with "handle_redirect returning false" do + before do + expect(relative_location).to receive(:handle_redirect).and_return(false) + end + + it "should not follow the redirect" do + response = relative_location.get('/') + response.finish + + expect(response).to be(:redirection?) end end end From e8d1b82045b95bac918074f04a1ba3f27b7b75e0 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 14 Aug 2024 14:30:05 +1200 Subject: [PATCH 038/125] Move `Async::HTTP::RelativeLocation` to `Async::HTTP::Middleware::LocationRedirector`. (#174) --- .../http/middleware/location_redirector.rb | 144 ++++++++++++++++++ lib/async/http/relative_location.rb | 136 +---------------- .../location_redirector.rb} | 6 +- 3 files changed, 152 insertions(+), 134 deletions(-) create mode 100644 lib/async/http/middleware/location_redirector.rb rename test/async/http/{relative_location.rb => middleware/location_redirector.rb} (94%) diff --git a/lib/async/http/middleware/location_redirector.rb b/lib/async/http/middleware/location_redirector.rb new file mode 100644 index 0000000..932f24e --- /dev/null +++ b/lib/async/http/middleware/location_redirector.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2019-2020, by Brian Morearty. + +require_relative '../reference' + +require 'protocol/http/middleware' +require 'protocol/http/body/rewindable' + +module Async + module HTTP + module Middleware + # A client wrapper which transparently handles redirects to a given maximum number of hops. + # + # The default implementation will only follow relative locations (i.e. those without a scheme) and will switch to GET if the original request was not a GET. + # + # The best reference for these semantics is defined by the [Fetch specification](https://fetch.spec.whatwg.org/#http-redirect-fetch). + # + # | Redirect using GET | Permanent | Temporary | + # |:-----------------------------------------:|:---------:|:---------:| + # | Allowed | 301 | 302 | + # | Preserve original method | 308 | 307 | + # + # For the specific details of the redirect handling, see: + # - 301 Moved Permanently. + # - 302 Found. + # - 307 Temporary Redirect. + # + class LocationRedirector < ::Protocol::HTTP::Middleware + class TooManyRedirects < StandardError + end + + # Header keys which should be deleted when changing a request from a POST to a GET as defined by . + PROHIBITED_GET_HEADERS = [ + 'content-encoding', + 'content-language', + 'content-location', + 'content-type', + ] + + # maximum_hops is the max number of redirects. Set to 0 to allow 1 request with no redirects. + def initialize(app, maximum_hops = 3) + super(app) + + @maximum_hops = maximum_hops + end + + # The maximum number of hops which will limit the number of redirects until an error is thrown. + attr :maximum_hops + + def redirect_with_get?(request, response) + # We only want to switch to GET if the request method is something other than get, e.g. POST. + if request.method != GET + # According to the RFC, we should only switch to GET if the response is a 301 or 302: + return response.status == 301 || response.status == 302 + end + end + + # Handle a redirect to a relative location. + # + # @parameter request [Protocol::HTTP::Request] The original request, which you can modify if you want to handle the redirect. + # @parameter location [String] The relative location to redirect to. + # @returns [Boolean] True if the redirect was handled, false if it was not. + def handle_redirect(request, location) + uri = URI.parse(location) + + if uri.absolute? + return false + end + + # Update the path of the request: + request.path = Reference[request.path] + location + + # Follow the redirect: + return true + end + + def call(request) + # We don't want to follow redirects for HEAD requests: + return super if request.head? + + if body = request.body + if body.respond_to?(:rewind) + # The request body was already rewindable, so use it as is: + body = request.body + else + # The request body was not rewindable, and we might need to resubmit it if we get a response status of 307 or 308, so make it rewindable: + body = ::Protocol::HTTP::Body::Rewindable.new(body) + request.body = body + end + end + + hops = 0 + + while hops <= @maximum_hops + response = super(request) + + if response.redirection? + hops += 1 + + # Get the redirect location: + unless location = response.headers['location'] + return response + end + + response.finish + + unless handle_redirect(request, location) + return response + end + + # Ensure the request (body) is finished and set to nil before we manipulate the request: + request.finish + + if request.method == GET or response.preserve_method? + # We (might) need to rewind the body so that it can be submitted again: + body&.rewind + request.body = body + else + # We are changing the method to GET: + request.method = GET + + # We will no longer be submitting the body: + body = nil + + # Remove any headers which are not allowed in a GET request: + PROHIBITED_GET_HEADERS.each do |header| + request.headers.delete(header) + end + end + else + return response + end + end + + raise TooManyRedirects, "Redirected #{hops} times, exceeded maximum!" + end + end + end + end +end \ No newline at end of file diff --git a/lib/async/http/relative_location.rb b/lib/async/http/relative_location.rb index 16f969f..884e297 100644 --- a/lib/async/http/relative_location.rb +++ b/lib/async/http/relative_location.rb @@ -4,141 +4,15 @@ # Copyright, 2018-2023, by Samuel Williams. # Copyright, 2019-2020, by Brian Morearty. -require_relative 'client' -require_relative 'endpoint' -require_relative 'reference' +require_relative 'middleware/location_redirector' -require 'protocol/http/middleware' -require 'protocol/http/body/rewindable' +warn "`Async::HTTP::RelativeLocation` is deprecated and will be removed in the next release. Please use `Async::HTTP::Middleware::LocationRedirector` instead.", uplevel: 1 module Async module HTTP - class TooManyRedirects < StandardError - end - - # A client wrapper which transparently handles redirects to a given maximum number of hops. - # - # The default implementation will only follow relative locations (i.e. those without a scheme) and will switch to GET if the original request was not a GET. - # - # The best reference for these semantics is defined by the [Fetch specification](https://fetch.spec.whatwg.org/#http-redirect-fetch). - # - # | Redirect using GET | Permanent | Temporary | - # |:-----------------------------------------:|:---------:|:---------:| - # | Allowed | 301 | 302 | - # | Preserve original method | 308 | 307 | - # - # For the specific details of the redirect handling, see: - # - 301 Moved Permanently. - # - 302 Found. - # - 307 Temporary Redirect. - # - class RelativeLocation < ::Protocol::HTTP::Middleware - # Header keys which should be deleted when changing a request from a POST to a GET as defined by . - PROHIBITED_GET_HEADERS = [ - 'content-encoding', - 'content-language', - 'content-location', - 'content-type', - ] - - # maximum_hops is the max number of redirects. Set to 0 to allow 1 request with no redirects. - def initialize(app, maximum_hops = 3) - super(app) - - @maximum_hops = maximum_hops - end - - # The maximum number of hops which will limit the number of redirects until an error is thrown. - attr :maximum_hops - - def redirect_with_get?(request, response) - # We only want to switch to GET if the request method is something other than get, e.g. POST. - if request.method != GET - # According to the RFC, we should only switch to GET if the response is a 301 or 302: - return response.status == 301 || response.status == 302 - end - end - - # Handle a redirect to a relative location. - # - # @parameter request [Protocol::HTTP::Request] The original request, which you can modify if you want to handle the redirect. - # @parameter location [String] The relative location to redirect to. - # @returns [Boolean] True if the redirect was handled, false if it was not. - def handle_redirect(request, location) - uri = URI.parse(location) - - if uri.absolute? - return false - end - - # Update the path of the request: - request.path = Reference[request.path] + location - - # Follow the redirect: - return true - end - - def call(request) - # We don't want to follow redirects for HEAD requests: - return super if request.head? - - if body = request.body - if body.respond_to?(:rewind) - # The request body was already rewindable, so use it as is: - body = request.body - else - # The request body was not rewindable, and we might need to resubmit it if we get a response status of 307 or 308, so make it rewindable: - body = ::Protocol::HTTP::Body::Rewindable.new(body) - request.body = body - end - end - - hops = 0 - - while hops <= @maximum_hops - response = super(request) - - if response.redirection? - hops += 1 - - # Get the redirect location: - unless location = response.headers['location'] - return response - end - - response.finish - - unless handle_redirect(request, location) - return response - end - - # Ensure the request (body) is finished and set to nil before we manipulate the request: - request.finish - - if request.method == GET or response.preserve_method? - # We (might) need to rewind the body so that it can be submitted again: - body&.rewind - request.body = body - else - # We are changing the method to GET: - request.method = GET - - # We will no longer be submitting the body: - body = nil - - # Remove any headers which are not allowed in a GET request: - PROHIBITED_GET_HEADERS.each do |header| - request.headers.delete(header) - end - end - else - return response - end - end - - raise TooManyRedirects, "Redirected #{hops} times, exceeded maximum!" - end + module Middleware + RelativeLocation = Middleware::LocationRedirector + TooManyRedirects = RelativeLocation::TooManyRedirects end end end diff --git a/test/async/http/relative_location.rb b/test/async/http/middleware/location_redirector.rb similarity index 94% rename from test/async/http/relative_location.rb rename to test/async/http/middleware/location_redirector.rb index 6d8c135..c738d74 100644 --- a/test/async/http/relative_location.rb +++ b/test/async/http/middleware/location_redirector.rb @@ -4,12 +4,12 @@ # Copyright, 2018-2023, by Samuel Williams. # Copyright, 2019-2020, by Brian Morearty. -require 'async/http/relative_location' +require 'async/http/middleware/location_redirector' require 'async/http/server' require 'sus/fixtures/async/http' -describe Async::HTTP::RelativeLocation do +describe Async::HTTP::Middleware::LocationRedirector do include Sus::Fixtures::Async::HTTP::ServerContext let(:relative_location) {subject.new(@client, 1)} @@ -49,7 +49,7 @@ it 'should fail with maximum redirects' do expect do response = relative_location.get('/home') - end.to raise_exception(Async::HTTP::TooManyRedirects, message: be =~ /maximum/) + end.to raise_exception(subject::TooManyRedirects, message: be =~ /maximum/) end end From b14afb7435d1754271d25380f121e4da8bd06da4 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 14 Aug 2024 14:34:31 +1200 Subject: [PATCH 039/125] Whitespace. --- lib/async/http/protocol/http1/server.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 6480ee1..68c3400 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -60,9 +60,9 @@ def each(task: Task.current) # If a response was generated, send it: if response trailer = response.headers.trailer! - + write_response(@version, response.status, response.headers) - + # Some operations in this method are long running, that is, it's expected that `body.call(stream)` could literally run indefinitely. In order to facilitate garbage collection, we want to nullify as many local variables before calling the streaming body. This ensures that the garbage collection can clean up as much state as possible during the long running operation, so we don't retain objects that are no longer needed. if body and protocol = response.protocol @@ -89,7 +89,7 @@ def each(task: Task.current) write_body(version, body, head, trailer) end - + # We are done with the body, you shouldn't need to call close on it: body = nil else From d61fb8f5032ee3ca539179ce3d8e05f2998d60f8 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 14 Aug 2024 14:36:02 +1200 Subject: [PATCH 040/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 5f22c06..88f8a5e 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.69.0" + VERSION = "0.70.0" end end From 7dde0bc5b5f3752d6b3ff1b365b8d096e0b88d96 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 26 Aug 2024 19:24:25 +1200 Subject: [PATCH 041/125] Slightly improved error handling. --- lib/async/http/protocol/http1/server.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 68c3400..24ed951 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -7,6 +7,7 @@ # Copyright, 2024, by Anton Zhuravsky. require_relative 'connection' +require 'console/event/failure' module Async module HTTP @@ -17,8 +18,9 @@ def fail_request(status) @persistent = false write_response(@version, status, {}) write_body(@version, nil) - rescue Errno::ECONNRESET, Errno::EPIPE - # Nothing we can do... + rescue => error + # At this point, there is very little we can do to recover: + Console::Event::Failure.for(error).emit(self, "Failed to write failure response.", severity: :debug) end def next_request @@ -33,7 +35,7 @@ def next_request end return request - rescue Async::TimeoutError + rescue Async::TimeoutError, IO::TimeoutError # For an interesting discussion about this behaviour, see https://trac.nginx.org/nginx/ticket/1005 # If you enable this, you will see some spec failures... # fail_request(408) From 598bd829b831a04fd3da32f6fce8d479f1544e62 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 26 Aug 2024 20:00:22 +1200 Subject: [PATCH 042/125] More specific `BadRequest` error handling. --- async-http.gemspec | 2 +- lib/async/http/protocol/http1/server.rb | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index ec72274..74b150e 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -29,7 +29,7 @@ Gem::Specification.new do |spec| spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" spec.add_dependency "protocol-http", "~> 0.28" - spec.add_dependency "protocol-http1", "~> 0.19" + spec.add_dependency "protocol-http1", "~> 0.20" spec.add_dependency "protocol-http2", "~> 0.18" spec.add_dependency "traces", ">= 0.10" end diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 24ed951..b7acbcf 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -35,13 +35,9 @@ def next_request end return request - rescue Async::TimeoutError, IO::TimeoutError - # For an interesting discussion about this behaviour, see https://trac.nginx.org/nginx/ticket/1005 - # If you enable this, you will see some spec failures... - # fail_request(408) - raise - rescue + rescue ::Protocol::HTTP1::BadRequest fail_request(400) + # Conceivably we could retry here, but we don't really know how bad the error is, so it's better to just fail: raise end From fe9586cd111755475c26bb7eb53dc6f84473fe72 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 26 Aug 2024 20:02:08 +1200 Subject: [PATCH 043/125] Modernize gem. --- .rubocop.yml | 3 +++ lib/async/http/middleware/location_redirector.rb | 5 ++--- lib/async/http/relative_location.rb | 2 +- readme.md | 16 ++++++++-------- .../async/http/middleware/location_redirector.rb | 2 +- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 442c667..a2447c2 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -16,6 +16,9 @@ Layout/IndentationConsistency: Enabled: true EnforcedStyle: normal +Layout/BlockAlignment: + Enabled: true + Layout/EndAlignment: Enabled: true EnforcedStyleAlignWith: start_of_line diff --git a/lib/async/http/middleware/location_redirector.rb b/lib/async/http/middleware/location_redirector.rb index 932f24e..41a7281 100644 --- a/lib/async/http/middleware/location_redirector.rb +++ b/lib/async/http/middleware/location_redirector.rb @@ -1,8 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. -# Copyright, 2019-2020, by Brian Morearty. +# Copyright, 2024, by Samuel Williams. require_relative '../reference' @@ -141,4 +140,4 @@ def call(request) end end end -end \ No newline at end of file +end diff --git a/lib/async/http/relative_location.rb b/lib/async/http/relative_location.rb index 884e297..e99d383 100644 --- a/lib/async/http/relative_location.rb +++ b/lib/async/http/relative_location.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. # Copyright, 2019-2020, by Brian Morearty. require_relative 'middleware/location_redirector' diff --git a/readme.md b/readme.md index 076e223..a636dff 100644 --- a/readme.md +++ b/readme.md @@ -12,6 +12,14 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f - [Testing](https://socketry.github.io/async-http/guides/testing/index) - This guide explains how to use `Async::HTTP` clients and servers in your tests. +## See Also + + - [benchmark-http](https://github.com/socketry/benchmark-http) — A benchmarking tool to report on web server concurrency. + - [falcon](https://github.com/socketry/falcon) — A rack compatible server built on top of `async-http`. + - [async-websocket](https://github.com/socketry/async-websocket) — Asynchronous client and server websockets. + - [async-rest](https://github.com/socketry/async-rest) — A RESTful resource layer built on top of `async-http`. + - [async-http-faraday](https://github.com/socketry/async-http-faraday) — A faraday adapter to use `async-http`. + ## Contributing We welcome contributions to this project. @@ -22,14 +30,6 @@ We welcome contributions to this project. 4. Push to the branch (`git push origin my-new-feature`). 5. Create new Pull Request. -## See Also - - - [benchmark-http](https://github.com/socketry/benchmark-http) — A benchmarking tool to report on web server concurrency. - - [falcon](https://github.com/socketry/falcon) — A rack compatible server built on top of `async-http`. - - [async-websocket](https://github.com/socketry/async-websocket) — Asynchronous client and server websockets. - - [async-rest](https://github.com/socketry/async-rest) — A RESTful resource layer built on top of `async-http`. - - [async-http-faraday](https://github.com/socketry/async-http-faraday) — A faraday adapter to use `async-http`. - ### Developer Certificate of Origin In order to protect users of this project, we require all contributors to comply with the [Developer Certificate of Origin](https://developercertificate.org/). This ensures that all contributions are properly licensed and attributed. diff --git a/test/async/http/middleware/location_redirector.rb b/test/async/http/middleware/location_redirector.rb index c738d74..1efb8ee 100644 --- a/test/async/http/middleware/location_redirector.rb +++ b/test/async/http/middleware/location_redirector.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. # Copyright, 2019-2020, by Brian Morearty. require 'async/http/middleware/location_redirector' From d434a21fe5ab8eaa7b9a5b6b2bb6e5994428af60 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 26 Aug 2024 20:04:57 +1200 Subject: [PATCH 044/125] Fix type syntax. --- lib/async/http/endpoint.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index 9006a3e..4fba7d7 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -62,7 +62,7 @@ def self.[](url) # @option hostname [String] the hostname to connect to (or bind to), overrides the URL hostname (used for SNI). # @option port [Integer] the port to bind to, overrides the URL port. # @option ssl_context [OpenSSL::SSL::SSLContext] the context to use for TLS. - # @option alpn_protocols [Array] the alpn protocols to negotiate. + # @option alpn_protocols [Array(String)] the alpn protocols to negotiate. def initialize(url, endpoint = nil, **options) super(**options) From a0f06f0dd4fff75a8eb71aeccb861726b850bf31 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 26 Aug 2024 20:07:30 +1200 Subject: [PATCH 045/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 88f8a5e..4a04705 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.70.0" + VERSION = "0.71.0" end end From f340dc8c7e6b75e07f5e3e8bbbdb33d65f21595b Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 26 Aug 2024 22:27:06 +1200 Subject: [PATCH 046/125] Add `http.version` to client/server traces. --- .github/workflows/test.yaml | 2 ++ lib/async/http/client.rb | 4 ++++ lib/async/http/server.rb | 1 + 3 files changed, 7 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0769a98..b9f70e9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,6 +7,8 @@ permissions: env: CONSOLE_OUTPUT: XTerm + TRACES_BACKEND: traces/backend/test + METRICS_BACKEND: metrics/backend/test jobs: test: diff --git a/lib/async/http/client.rb b/lib/async/http/client.rb index 58ebfc2..bbaeb18 100755 --- a/lib/async/http/client.rb +++ b/lib/async/http/client.rb @@ -165,6 +165,10 @@ def call(request) end super.tap do |response| + if version = response&.version + span['http.version'] = version + end + if status = response&.status span['http.status_code'] = status end diff --git a/lib/async/http/server.rb b/lib/async/http/server.rb index 311bd61..702d393 100755 --- a/lib/async/http/server.rb +++ b/lib/async/http/server.rb @@ -81,6 +81,7 @@ def call(request) end attributes = { + 'http.version': request.version, 'http.method': request.method, 'http.authority': request.authority, 'http.scheme': request.scheme, From f575ddb739452e1042edf2b02d417500f870d721 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 29 Aug 2024 10:53:08 +1200 Subject: [PATCH 047/125] Use `Rewindable#wrap`. --- async-http.gemspec | 2 +- lib/async/http/middleware/location_redirector.rb | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index 74b150e..59d2545 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.7" spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" - spec.add_dependency "protocol-http", "~> 0.28" + spec.add_dependency "protocol-http", "~> 0.29" spec.add_dependency "protocol-http1", "~> 0.20" spec.add_dependency "protocol-http2", "~> 0.18" spec.add_dependency "traces", ">= 0.10" diff --git a/lib/async/http/middleware/location_redirector.rb b/lib/async/http/middleware/location_redirector.rb index 41a7281..ae8c24f 100644 --- a/lib/async/http/middleware/location_redirector.rb +++ b/lib/async/http/middleware/location_redirector.rb @@ -81,17 +81,7 @@ def call(request) # We don't want to follow redirects for HEAD requests: return super if request.head? - if body = request.body - if body.respond_to?(:rewind) - # The request body was already rewindable, so use it as is: - body = request.body - else - # The request body was not rewindable, and we might need to resubmit it if we get a response status of 307 or 308, so make it rewindable: - body = ::Protocol::HTTP::Body::Rewindable.new(body) - request.body = body - end - end - + body = ::Protocol::HTTP::Body::Rewindable.wrap(request) hops = 0 while hops <= @maximum_hops From 718d83c1bb288ca4a79a611f52c4b3020167af67 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 29 Aug 2024 12:51:12 +1200 Subject: [PATCH 048/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 4a04705..dd5222c 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.71.0" + VERSION = "0.72.0" end end From 91c2fff15395ef3ec80d2903a4eca8ee4feccccd Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 31 Aug 2024 09:30:17 +1200 Subject: [PATCH 049/125] Correctly specify headers as a hash. --- test/async/http/client/google.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/async/http/client/google.rb b/test/async/http/client/google.rb index cc1f70f..cc9c1d6 100644 --- a/test/async/http/client/google.rb +++ b/test/async/http/client/google.rb @@ -22,7 +22,7 @@ end it 'can fetch remote resource' do - response = client.get('/', 'accept' => '*/*') + response = client.get('/', {'accept' => '*/*'}) response.finish From f4d82c5be42672d3338b57d2fb90807675766e52 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 31 Aug 2024 15:25:16 +1200 Subject: [PATCH 050/125] Updated (client) support for interim responses. --- async-http.gemspec | 2 +- fixtures/async/http/a_protocol.rb | 18 ++++++++--- lib/async/http/protocol/http1/request.rb | 6 ++-- lib/async/http/protocol/http1/response.rb | 2 ++ lib/async/http/protocol/http2/request.rb | 20 ++++++------ lib/async/http/protocol/http2/response.rb | 37 +++++++++++++++++------ lib/async/http/protocol/http2/stream.rb | 3 +- lib/async/http/protocol/request.rb | 2 +- 8 files changed, 59 insertions(+), 31 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index 59d2545..a69e021 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.7" spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" - spec.add_dependency "protocol-http", "~> 0.29" + spec.add_dependency "protocol-http", "~> 0.30" spec.add_dependency "protocol-http1", "~> 0.20" spec.add_dependency "protocol-http2", "~> 0.18" spec.add_dependency "traces", ">= 0.10" diff --git a/fixtures/async/http/a_protocol.rb b/fixtures/async/http/a_protocol.rb index 6d7654f..8fbd5e2 100644 --- a/fixtures/async/http/a_protocol.rb +++ b/fixtures/async/http/a_protocol.rb @@ -44,18 +44,28 @@ module HTTP with "interim response" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| - request.write_interim_response( - ::Protocol::HTTP::Response[103, [["link", "; rel=preload; as=style"]]] - ) + request.send_interim_response(103, [["link", "; rel=preload; as=style"]]) ::Protocol::HTTP::Response[200, {}, ["Hello World"]] end end it "can read informational response" do - response = client.get("/") + called = false + + callback = proc do |status, headers| + called = true + expect(status).to be == 103 + expect(headers).to have_keys( + "link" => be == ["; rel=preload; as=style"] + ) + end + + response = client.get("/", interim_response: callback) expect(response).to be(:success?) expect(response.read).to be == "Hello World" + + expect(called).to be == true end end diff --git a/lib/async/http/protocol/http1/request.rb b/lib/async/http/protocol/http1/request.rb index 3e129c6..d70ee3e 100644 --- a/lib/async/http/protocol/http1/request.rb +++ b/lib/async/http/protocol/http1/request.rb @@ -24,7 +24,7 @@ def initialize(connection, authority, method, path, version, headers, body) # HTTP/1 requests with an upgrade header (which can contain zero or more values) are extracted into the protocol field of the request, and we expect a response to select one of those protocols with a status code of 101 Switching Protocols. protocol = headers.delete('upgrade') - super(nil, authority, method, path, version, headers, body, protocol) + super(nil, authority, method, path, version, headers, body, protocol, self.public_method(:write_interim_response)) end def connection @@ -39,8 +39,8 @@ def hijack! @connection.hijack! end - def write_interim_response(response) - @connection.write_interim_response(response.version, response.status, response.headers) + def write_interim_response(status, headers = nil) + @connection.write_interim_response(@version, status, headers) end end end diff --git a/lib/async/http/protocol/http1/response.rb b/lib/async/http/protocol/http1/response.rb index 7f30464..a7a8c6f 100644 --- a/lib/async/http/protocol/http1/response.rb +++ b/lib/async/http/protocol/http1/response.rb @@ -17,6 +17,8 @@ def self.read(connection, request) if response.final? return response + else + request.send_interim_response(response.status, response.headers) end end end diff --git a/lib/async/http/protocol/http2/request.rb b/lib/async/http/protocol/http2/request.rb index 4fe519d..60e04d1 100644 --- a/lib/async/http/protocol/http2/request.rb +++ b/lib/async/http/protocol/http2/request.rb @@ -23,6 +23,8 @@ def initialize(*) attr :request def receive_initial_headers(headers, end_stream) + @headers = ::Protocol::HTTP::Headers.new + headers.each do |key, value| if key == SCHEME raise ::Protocol::HTTP2::HeaderError, "Request scheme already specified!" if @request.scheme @@ -85,7 +87,7 @@ def closed(error) end def initialize(stream) - super(nil, nil, nil, nil, VERSION, nil) + super(nil, nil, nil, nil, VERSION, nil, nil, nil, self.public_method(:write_interim_response)) @stream = stream end @@ -117,10 +119,6 @@ def send_response(response) [STATUS, response.status], ] - if protocol = response.protocol - protocol_headers << [PROTOCOL, protocol] - end - if length = response.body&.length protocol_headers << [CONTENT_LENGTH, length] end @@ -142,14 +140,16 @@ def send_response(response) end end - def write_interim_response(response) - protocol_headers = [ - [STATUS, response.status] + def write_interim_response(status, headers = nil) + interim_response_headers = [ + [STATUS, status] ] - headers = ::Protocol::HTTP::Headers::Merged.new(protocol_headers, response.headers) + if headers + interim_response_headers = ::Protocol::HTTP::Headers::Merged.new(interim_response_headers, headers) + end - @stream.send_headers(nil, headers) + @stream.send_headers(nil, interim_response_headers) end end end diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index fccebcb..13e0c41 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -38,18 +38,25 @@ def accept_push_promise_stream(promised_stream_id, headers) # This should be invoked from the background reader, and notifies the task waiting for the headers that we are done. def receive_initial_headers(headers, end_stream) + # While in theory, the response pseudo-headers may be extended in the future, currently they only response pseudo-header is :status, so we can assume it is always the first header. + status_header = headers.shift + + if status_header.first != ':status' + raise ProtocolError, "Invalid response headers: #{headers.inspect}" + end + + status = Integer(status_header.last) + + if status >= 100 && status < 200 + return receive_interim_headers(status, headers) + end + + @response.status = status + @headers = ::Protocol::HTTP::Headers.new + headers.each do |key, value| # It's guaranteed that this should be the first header: - if key == STATUS - status = Integer(value) - - # Ignore informational headers: - return if status >= 100 && status < 200 - - @response.status = Integer(value) - elsif key == PROTOCOL - @response.protocol = value - elsif key == CONTENT_LENGTH + if key == CONTENT_LENGTH @length = Integer(value) else add_header(key, value) @@ -74,6 +81,16 @@ def receive_initial_headers(headers, end_stream) return headers end + def receive_interim_headers(status, headers) + if headers.any? + headers = ::Protocol::HTTP::Headers[headers] + else + headers = nil + end + + @response.request.send_interim_response(status, headers) + end + # Notify anyone waiting on the response headers to be received (or failure). def notify! if notification = @notification diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index 580ed19..49d2587 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -51,10 +51,9 @@ def receive_trailing_headers(headers, end_stream) end def process_headers(frame) - if frame.end_stream? && @headers + if @headers and frame.end_stream? self.receive_trailing_headers(super, frame.end_stream?) else - @headers ||= ::Protocol::HTTP::Headers.new self.receive_initial_headers(super, frame.end_stream?) end diff --git a/lib/async/http/protocol/request.rb b/lib/async/http/protocol/request.rb index 6782718..41c7f29 100644 --- a/lib/async/http/protocol/request.rb +++ b/lib/async/http/protocol/request.rb @@ -25,7 +25,7 @@ def hijack? false end - def write_interim_response(response) + def write_interim_response(status, headers = nil) end def peer From 3eb6983ddcf423c620cde8d004a01da2565f8cb4 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 31 Aug 2024 22:50:10 +1200 Subject: [PATCH 051/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index dd5222c..bd2617d 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.72.0" + VERSION = "0.73.0" end end From f3e6a3689694ca079274a3f539c359daed256bea Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 1 Sep 2024 10:38:47 +1200 Subject: [PATCH 052/125] Update `Internet` to accept keyword arguments. --- lib/async/http/internet.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/async/http/internet.rb b/lib/async/http/internet.rb index 37ee38f..0a3622a 100644 --- a/lib/async/http/internet.rb +++ b/lib/async/http/internet.rb @@ -39,14 +39,14 @@ def client_for(endpoint) # @parameter url [String] The URL to request, e.g. `https://www.codeotaku.com`. # @parameter headers [Hash | Protocol::HTTP::Headers] The headers to send with the request. # @parameter body [String | Protocol::HTTP::Body] The body to send with the request. - def call(method, url, headers = nil, body = nil, &block) + def call(verb, url, *arguments, **options, &block) endpoint = Endpoint[url] client = self.client_for(endpoint) - body = Body::Buffered.wrap(body) - headers = ::Protocol::HTTP::Headers[headers] + options[:authority] ||= endpoint.authority + options[:scheme] ||= endpoint.scheme - request = ::Protocol::HTTP::Request.new(endpoint.scheme, endpoint.authority, method, endpoint.path, nil, headers, body) + request = ::Protocol::HTTP::Request[verb, endpoint.path, *arguments, **options] response = client.call(request) @@ -68,8 +68,8 @@ def close end ::Protocol::HTTP::Methods.each do |name, verb| - define_method(verb.downcase) do |url, headers = nil, body = nil, &block| - self.call(verb, url, headers, body, &block) + define_method(verb.downcase) do |url, *arguments, **options, &block| + self.call(verb, url, *arguments, **options, &block) end end From d5fecf9d1e43021a0edef7f71cf132b5d93368ac Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 1 Sep 2024 10:57:04 +1200 Subject: [PATCH 053/125] Add `releases.md`. --- bake.rb | 12 ++++++++++++ gems.rb | 2 +- readme.md | 12 ++++++++++++ releases.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 bake.rb create mode 100644 releases.md diff --git a/bake.rb b/bake.rb new file mode 100644 index 0000000..52b35d0 --- /dev/null +++ b/bake.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2024, by Samuel Williams. + +# Update the project documentation with the new version number. +# +# @parameter version [String] The new version number. +def after_gem_release_version_increment(version) + context['releases:update'].call(version) + context['utopia:project:readme:update'].call +end diff --git a/gems.rb b/gems.rb index 1fe0e80..4094e8a 100644 --- a/gems.rb +++ b/gems.rb @@ -24,8 +24,8 @@ gem "bake-modernize" gem "bake-gem" - gem "falcon", "~> 0.47" gem "utopia-project" + gem "bake-releases" end group :test do diff --git a/readme.md b/readme.md index a636dff..8edeb18 100644 --- a/readme.md +++ b/readme.md @@ -12,6 +12,18 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f - [Testing](https://socketry.github.io/async-http/guides/testing/index) - This guide explains how to use `Async::HTTP` clients and servers in your tests. +## Releases + +Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. + +### Unreleased + + - [`Async::HTTP::Internet` accepts keyword arguments](https://socketry.github.io/async-http/releases/index#async::http::internet-accepts-keyword-arguments) + +### v0.73.0 + + - [Update support for `interim_response`](https://socketry.github.io/async-http/releases/index#update-support-for-interim_response) + ## See Also - [benchmark-http](https://github.com/socketry/benchmark-http) — A benchmarking tool to report on web server concurrency. diff --git a/releases.md b/releases.md new file mode 100644 index 0000000..9211bef --- /dev/null +++ b/releases.md @@ -0,0 +1,51 @@ +# Releases + +## Unreleased + +### `Async::HTTP::Internet` accepts keyword arguments + +`Async::HTTP::Internet` now accepts keyword arguments for making a request, e.g. + +```ruby +internet = Async::HTTP::Internet.instance + +# This will let you override the authority (HTTP/1.1 host header, HTTP/2 :authority header): +internet.get("https://proxy.local", authority: "example.com") + +# This will let you override the scheme: +internet.get("https://example.com", scheme: "http") +``` + +## v0.73.0 + +### Update support for `interim_response` + +`Protocol::HTTP::Request` now supports an `interim_response` callback, which will be called with the interim response status and headers. This works on both the client and the server: + +```ruby +# Server side: +def call(request) + if request.headers['expect'].include?('100-continue') + request.send_interim_response(100) + end + + # ... +end + +# Client side: +body = Async::HTTP::Body::Writable.new + +interim_repsonse = proc do |status, headers| + if status == 100 + # Continue sending the body... + body.write("Hello, world!") + body.close + end +end + +Async::HTTP::Internet.instance.post("https://example.com", body, interim_response: interim_response) do |response| + unless response.success? + body.close + end +end +``` From 529c3a998d8917b7cfb090a4d53b30c2387ef5fd Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 1 Sep 2024 11:25:50 +1200 Subject: [PATCH 054/125] Fix race conditions in trailer tests. --- fixtures/async/http/a_protocol.rb | 53 +++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/fixtures/async/http/a_protocol.rb b/fixtures/async/http/a_protocol.rb index 8fbd5e2..09b136e 100644 --- a/fixtures/async/http/a_protocol.rb +++ b/fixtures/async/http/a_protocol.rb @@ -5,6 +5,7 @@ # Copyright, 2020, by Igor Sidorov. require 'async' +require 'async/variable' require 'async/clock' require 'async/http/client' require 'async/http/server' @@ -128,29 +129,22 @@ module HTTP end end - with 'with trailer' do + with 'with request trailer' do + let(:request_received) {Async::Variable.new} + let(:app) do ::Protocol::HTTP::Middleware.for do |request| if trailer = request.headers['trailer'] expect(request.headers).not.to have_keys('etag') + + request_received.value = true request.finish + expect(request.headers).to have_keys('etag') ::Protocol::HTTP::Response[200, [], "request trailer"] else - headers = ::Protocol::HTTP::Headers.new - headers.add('trailer', 'etag') - - body = Async::HTTP::Body::Writable.new - - Async do |task| - body.write("response trailer") - sleep(0.01) - headers.add('etag', 'abcd') - body.close - end - - ::Protocol::HTTP::Response[200, headers, body] + ::Protocol::HTTP::Response[400, headers, body] end end end @@ -164,16 +158,41 @@ module HTTP Async do |task| body.write("Hello") - sleep(0.01) + + request_received.wait headers.add('etag', 'abcd') + body.close end response = client.post("/", headers, body) expect(response.read).to be == "request trailer" - expect(response).to be(:success?) end + end + + with "with response trailer" do + let(:response_received) {Async::Variable.new} + + let(:app) do + ::Protocol::HTTP::Middleware.for do |request| + headers = ::Protocol::HTTP::Headers.new + headers.add('trailer', 'etag') + + body = Async::HTTP::Body::Writable.new + + Async do |task| + body.write("response trailer") + + response_received.wait + headers.add('etag', 'abcd') + + body.close + end + + ::Protocol::HTTP::Response[200, headers, body] + end + end it "can receive response trailer" do skip "Protocol does not support trailers!" unless subject.bidirectional? @@ -183,6 +202,8 @@ module HTTP headers = response.headers expect(headers).not.to have_keys('etag') + response_received.value = true + expect(response.read).to be == "response trailer" expect(response).to be(:success?) From 67aa1dd58a3981cbf4135acf982a6589a7e87723 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 1 Sep 2024 11:36:57 +1200 Subject: [PATCH 055/125] Bump minor version. --- lib/async/http/version.rb | 2 +- readme.md | 2 +- releases.md | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index bd2617d..fbce57c 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.73.0" + VERSION = "0.74.0" end end diff --git a/readme.md b/readme.md index 8edeb18..8e37e82 100644 --- a/readme.md +++ b/readme.md @@ -16,7 +16,7 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. -### Unreleased +### v0.74.0 - [`Async::HTTP::Internet` accepts keyword arguments](https://socketry.github.io/async-http/releases/index#async::http::internet-accepts-keyword-arguments) diff --git a/releases.md b/releases.md index 9211bef..e1ca9b2 100644 --- a/releases.md +++ b/releases.md @@ -1,12 +1,12 @@ # Releases -## Unreleased +## v0.74.0 ### `Async::HTTP::Internet` accepts keyword arguments `Async::HTTP::Internet` now accepts keyword arguments for making a request, e.g. -```ruby +``` ruby internet = Async::HTTP::Internet.instance # This will let you override the authority (HTTP/1.1 host header, HTTP/2 :authority header): @@ -22,7 +22,7 @@ internet.get("https://example.com", scheme: "http") `Protocol::HTTP::Request` now supports an `interim_response` callback, which will be called with the interim response status and headers. This works on both the client and the server: -```ruby +``` ruby # Server side: def call(request) if request.headers['expect'].include?('100-continue') From 8523d4faab7ac4bf40958d7b610c8d4ed6724881 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 2 Sep 2024 13:32:04 +1200 Subject: [PATCH 056/125] Simpler annotations. --- lib/async/http/protocol/http1/client.rb | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index d53122d..2ee4d26 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -33,22 +33,16 @@ def call(request, task: Task.current) if protocol = request.protocol # This is a very tricky apect of handling HTTP/1 upgrade connections. In theory, this approach is a bit inefficient, because we spin up a task just to handle writing to the underlying stream when we could be writing to the stream directly. But we need to maintain some level of compatibility with HTTP/2. Additionally, we don't know if the upgrade request will be accepted, so starting to write the body at this point needs to be handled with care. - task.async do |subtask| - subtask.annotate("Upgrading request.") - + task.async(annotation: "Upgrading request...") do # If this fails, this connection will be closed. write_upgrade_body(protocol, body) end elsif request.connect? - task.async do |subtask| - subtask.annotate("Tunnelling body.") - + task.async(annotation: "Tunnneling request...") do write_tunnel_body(@version, body) end else - task.async do |subtask| - subtask.annotate("Streaming body.") - + task.async(annotation: "Streaming request...") do # Once we start writing the body, we can't recover if the request fails. That's because the body might be generated dynamically, streaming, etc. write_body(@version, body, false, trailer) end From fdc8906da309fb9400e2d6bf59ef1934607f2a94 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 4 Sep 2024 12:22:17 +1200 Subject: [PATCH 057/125] Better order of operations in `HTTP2::Connection#close`. --- lib/async/http/protocol/http2/connection.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/async/http/protocol/http2/connection.rb b/lib/async/http/protocol/http2/connection.rb index ccf931e..cbab8f7 100644 --- a/lib/async/http/protocol/http2/connection.rb +++ b/lib/async/http/protocol/http2/connection.rb @@ -66,14 +66,14 @@ def start_connection end def close(error = nil) - super - # Ensure the reader task is stopped. if @reader reader = @reader @reader = nil reader.stop end + + super end def read_in_background(parent: Task.current) @@ -101,6 +101,8 @@ def read_in_background(parent: Task.current) ensure # Don't call #close twice. if @reader + @reader = nil + self.close(error) end end From bdcc2ca450317a11dbf3f3bb0b50ccb3da4cefe8 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 4 Sep 2024 12:22:44 +1200 Subject: [PATCH 058/125] Prefer `self.` for clarity. --- lib/async/http/protocol/http2/response.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index 13e0c41..e38ea47 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -118,7 +118,7 @@ def closed(error) @exception = error - notify! + self.notify! end end From 105c1fc363f4266f9ff364982e627da59461dc4e Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 4 Sep 2024 12:24:21 +1200 Subject: [PATCH 059/125] Improve test robustness on failure case. We don't expect a response to be generated, but if it is, close it. --- fixtures/async/http/a_protocol.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fixtures/async/http/a_protocol.rb b/fixtures/async/http/a_protocol.rb index 09b136e..0dcde41 100644 --- a/fixtures/async/http/a_protocol.rb +++ b/fixtures/async/http/a_protocol.rb @@ -508,7 +508,9 @@ module HTTP it "can't get /" do expect do - client.get("/") + response = client.get("/") + ensure + response&.close end.to raise_exception(::IO::TimeoutError) end end From 54fa9b738499662bbcec6499eead5dc9d8690e6c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 4 Sep 2024 12:24:59 +1200 Subject: [PATCH 060/125] Ensure "upgrade" response use 101 status code. --- lib/async/http/protocol/http1/server.rb | 43 ++++++++++++++----------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index b7acbcf..91e1010 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -59,33 +59,40 @@ def each(task: Task.current) if response trailer = response.headers.trailer! - write_response(@version, response.status, response.headers) - # Some operations in this method are long running, that is, it's expected that `body.call(stream)` could literally run indefinitely. In order to facilitate garbage collection, we want to nullify as many local variables before calling the streaming body. This ensures that the garbage collection can clean up as much state as possible during the long running operation, so we don't retain objects that are no longer needed. - + if body and protocol = response.protocol + # We force a 101 response if the protocol is upgraded - HTTP/2 CONNECT will return 200 for success, but this won't be understood by HTTP/1 clients: + write_response(@version, 101, response.headers) + stream = write_upgrade_body(protocol) # At this point, the request body is hijacked, so we don't want to call #finish below. - request = response = nil - - body.call(stream) - elsif request.connect? and response.success? - stream = write_tunnel_body(request.version) - - # Same as above: - request = response = nil + request = nil unless request.body + response = nil body.call(stream) else - head = request.head? - version = request.version - - # Same as above: - request = nil unless request.body - response = nil + write_response(@version, response.status, response.headers) - write_body(version, body, head, trailer) + if request.connect? and response.success? + stream = write_tunnel_body(request.version) + + # Same as above: + request = nil unless request.body + response = nil + + body.call(stream) + else + head = request.head? + version = request.version + + # Same as above: + request = nil unless request.body + response = nil + + write_body(version, body, head, trailer) + end end # We are done with the body, you shouldn't need to call close on it: From 6f2d6e1ea6ac168e77c88ddf4919330e2bdb0faf Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 4 Sep 2024 12:25:13 +1200 Subject: [PATCH 061/125] For the sake of HTTP/1 proxy, copy the request protocol to the response. --- lib/async/http/protocol/http2/response.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index e38ea47..8c934a9 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -54,6 +54,11 @@ def receive_initial_headers(headers, end_stream) @response.status = status @headers = ::Protocol::HTTP::Headers.new + # If the protocol request was successful, ensure the response protocol matches: + if status == 200 and protocol = @response.request.protocol + @response.protocol = Array(protocol).first + end + headers.each do |key, value| # It's guaranteed that this should be the first header: if key == CONTENT_LENGTH From abfd8cff86d05d8c0798f765113964d13be7816e Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 4 Sep 2024 12:48:31 +1200 Subject: [PATCH 062/125] Ensure upgrade and tunnel responses correctly exit the request loop. --- lib/async/http/protocol/http1/server.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 91e1010..19078a6 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -71,7 +71,8 @@ def each(task: Task.current) request = nil unless request.body response = nil - body.call(stream) + # We must return here as no further request processing can be done: + return body.call(stream) else write_response(@version, response.status, response.headers) @@ -82,7 +83,8 @@ def each(task: Task.current) request = nil unless request.body response = nil - body.call(stream) + # We must return here as no further request processing can be done: + return body.call(stream) else head = request.head? version = request.version From 1d24862b2bb1fa41375973027d5189e1a36a31ff Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 4 Sep 2024 12:50:00 +1200 Subject: [PATCH 063/125] Modernize gem. --- .github/workflows/test-coverage.yaml | 4 ++-- bake.rb | 2 +- lib/async/http/protocol/http1/client.rb | 2 +- lib/async/http/protocol/http2/request.rb | 2 +- lib/async/http/protocol/http2/response.rb | 2 +- lib/async/http/protocol/request.rb | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index f9da2ff..ffa0927 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 5 run: bundle exec bake test - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: coverage-${{matrix.os}}-${{matrix.ruby}} path: .covered.db @@ -50,7 +50,7 @@ jobs: ruby-version: "3.3" bundler-cache: true - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 - name: Validate coverage timeout-minutes: 5 diff --git a/bake.rb b/bake.rb index 52b35d0..6971885 100644 --- a/bake.rb +++ b/bake.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2024, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. # Update the project documentation with the new version number. # diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index 2ee4d26..c7fa99b 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. require_relative 'connection' diff --git a/lib/async/http/protocol/http2/request.rb b/lib/async/http/protocol/http2/request.rb index 60e04d1..406ca2e 100644 --- a/lib/async/http/protocol/http2/request.rb +++ b/lib/async/http/protocol/http2/request.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. require_relative '../request' require_relative 'stream' diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index 8c934a9..5d79713 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. require_relative '../response' require_relative 'stream' diff --git a/lib/async/http/protocol/request.rb b/lib/async/http/protocol/request.rb index 41c7f29..ded0d2c 100644 --- a/lib/async/http/protocol/request.rb +++ b/lib/async/http/protocol/request.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2023, by Samuel Williams. +# Copyright, 2017-2024, by Samuel Williams. require 'protocol/http/request' require 'protocol/http/headers' From e462e45198f22052070d88968eb7b6a08d27b3bb Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 4 Sep 2024 12:54:57 +1200 Subject: [PATCH 064/125] Update `releases.md`. --- releases.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/releases.md b/releases.md index e1ca9b2..a527a13 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + +- Better handling of HTTP/1 <-> HTTP/2 proxying, specifically upgrade/CONNECT requests. + ## v0.74.0 ### `Async::HTTP::Internet` accepts keyword arguments From f9cef46767de469cf6af576952d32da465893fdf Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 4 Sep 2024 12:55:06 +1200 Subject: [PATCH 065/125] Bump minor version. --- lib/async/http/version.rb | 2 +- readme.md | 4 ++++ releases.md | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index fbce57c..4b9a3bd 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.74.0" + VERSION = "0.75.0" end end diff --git a/readme.md b/readme.md index 8e37e82..350f33a 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,10 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. +### v0.75.0 + + - Better handling of HTTP/1 \<-\> HTTP/2 proxying, specifically upgrade/CONNECT requests. + ### v0.74.0 - [`Async::HTTP::Internet` accepts keyword arguments](https://socketry.github.io/async-http/releases/index#async::http::internet-accepts-keyword-arguments) diff --git a/releases.md b/releases.md index a527a13..6bc17f8 100644 --- a/releases.md +++ b/releases.md @@ -1,8 +1,8 @@ # Releases -## Unreleased +## v0.75.0 -- Better handling of HTTP/1 <-> HTTP/2 proxying, specifically upgrade/CONNECT requests. + - Better handling of HTTP/1 \<-\> HTTP/2 proxying, specifically upgrade/CONNECT requests. ## v0.74.0 From de35b625211d029b68f0b338d895c618f4a90a34 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 6 Sep 2024 09:58:24 +1200 Subject: [PATCH 066/125] `Async::HTTP::Body::Writable` is moved to `Protocol::HTTP::Body::Writable`. --- async-http.gemspec | 2 +- lib/async/http/body/writable.rb | 95 +-------------------------------- 2 files changed, 3 insertions(+), 94 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index a69e021..c3a4df0 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.7" spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" - spec.add_dependency "protocol-http", "~> 0.30" + spec.add_dependency "protocol-http", "~> 0.33" spec.add_dependency "protocol-http1", "~> 0.20" spec.add_dependency "protocol-http2", "~> 0.18" spec.add_dependency "traces", ">= 0.10" diff --git a/lib/async/http/body/writable.rb b/lib/async/http/body/writable.rb index a86d4a8..18f7137 100644 --- a/lib/async/http/body/writable.rb +++ b/lib/async/http/body/writable.rb @@ -3,104 +3,13 @@ # Released under the MIT License. # Copyright, 2018-2023, by Samuel Williams. -require 'protocol/http/body/readable' +require 'protocol/http/body/writable' require 'async/queue' module Async module HTTP module Body - include ::Protocol::HTTP::Body - - # A dynamic body which you can write to and read from. - class Writable < Readable - class Closed < StandardError - end - - # @param [Integer] length The length of the response body if known. - # @param [Async::Queue] queue Specify a different queue implementation, e.g. `Async::LimitedQueue.new(8)` to enable back-pressure streaming. - def initialize(length = nil, queue: Async::Queue.new) - @queue = queue - - @length = length - - @count = 0 - - @finished = false - - @closed = false - @error = nil - end - - def length - @length - end - - # Stop generating output; cause the next call to write to fail with the given error. - def close(error = nil) - unless @closed - @queue.enqueue(nil) - - @closed = true - @error = error - end - - super - end - - def closed? - @closed - end - - def ready? - !@queue.empty? - end - - # Has the producer called #finish and has the reader consumed the nil token? - def empty? - @finished - end - - # Read the next available chunk. - def read - return if @finished - - unless chunk = @queue.dequeue - @finished = true - end - - return chunk - end - - # Write a single chunk to the body. Signal completion by calling `#finish`. - def write(chunk) - # If the reader breaks, the writer will break. - # The inverse of this is less obvious (*) - if @closed - raise(@error || Closed) - end - - @count += 1 - @queue.enqueue(chunk) - end - - alias << write - - def inspect - "\#<#{self.class} #{@count} chunks written, #{status}>" - end - - private - - def status - if @finished - 'finished' - elsif @closed - 'closing' - else - 'waiting' - end - end - end + Writable = ::Protocol::HTTP::Body::Writable end end end From b1c6cf6922cdbed3cd9df9a9f2f71258734033c5 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 6 Sep 2024 10:03:21 +1200 Subject: [PATCH 067/125] Remove race condition from test. --- fixtures/async/http/body/a_writable_body.rb | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/fixtures/async/http/body/a_writable_body.rb b/fixtures/async/http/body/a_writable_body.rb index eca1c5d..24c18c3 100644 --- a/fixtures/async/http/body/a_writable_body.rb +++ b/fixtures/async/http/body/a_writable_body.rb @@ -86,21 +86,16 @@ module Body }.to raise_exception(RuntimeError, message: be =~ /big/) end - it "will stop after finishing" do - output_task = reactor.async do - body.each do |chunk| - expect(chunk).to be == "Hello World!" - end - end - + it "can consume chunks" do body.write("Hello World!") body.close expect(body).not.to be(:empty?) - ::Async::Task.current.yield + body.each do |chunk| + expect(chunk).to be == "Hello World!" + end - expect(output_task).to be(:finished?) expect(body).to be(:empty?) end end From d5f0b319792e8e4e24e67efbbf09940a2546cc1b Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 10 Sep 2024 09:29:27 +1200 Subject: [PATCH 068/125] Tidy up body code. (#181) * Remove `Async::HTTP::Body::Delayed` with no replacement. * Remove `Async::HTTP::Body::Slowloris` with no replacement. * Fix handling of stream `close_write`. --- examples/upload/client.rb | 21 +++- fixtures/async/http/a_protocol.rb | 10 +- fixtures/async/http/body/a_writable_body.rb | 105 -------------------- gems.rb | 2 + lib/async/http/body/delayed.rb | 32 ------ lib/async/http/body/hijack.rb | 2 +- lib/async/http/body/pipe.rb | 10 +- lib/async/http/body/slowloris.rb | 55 ---------- lib/async/http/protocol/http1/server.rb | 16 ++- lib/async/http/protocol/http2/input.rb | 4 +- lib/async/http/protocol/http2/output.rb | 41 +++++--- lib/async/http/protocol/http2/stream.rb | 6 +- test/async/http/body.rb | 6 +- test/async/http/body/hijack.rb | 2 +- test/async/http/body/pipe.rb | 8 +- test/async/http/body/slowloris.rb | 35 ------- test/async/http/body/writable.rb | 17 ---- test/async/http/proxy.rb | 4 +- 18 files changed, 90 insertions(+), 286 deletions(-) delete mode 100644 fixtures/async/http/body/a_writable_body.rb delete mode 100644 lib/async/http/body/delayed.rb delete mode 100644 lib/async/http/body/slowloris.rb delete mode 100644 test/async/http/body/slowloris.rb delete mode 100644 test/async/http/body/writable.rb diff --git a/examples/upload/client.rb b/examples/upload/client.rb index 5d2c161..558581d 100644 --- a/examples/upload/client.rb +++ b/examples/upload/client.rb @@ -9,10 +9,27 @@ require 'async' require 'protocol/http/body/file' -require 'async/http/body/delayed' require 'async/http/client' require 'async/http/endpoint' +class Delayed < ::Protocol::HTTP::Body::Wrapper + def initialize(body, delay = 0.01) + super(body) + + @delay = delay + end + + def ready? + false + end + + def read + sleep(@delay) + + return super + end +end + Async do endpoint = Async::HTTP::Endpoint.parse("http://localhost:9222") client = Async::HTTP::Client.new(endpoint, protocol: Async::HTTP::Protocol::HTTP2) @@ -21,7 +38,7 @@ ['accept', 'text/plain'], ] - body = Async::HTTP::Body::Delayed.new(Protocol::HTTP::Body::File.open(File.join(__dir__, "data.txt"), block_size: 32)) + body = Delayed.new(Protocol::HTTP::Body::File.open(File.join(__dir__, "data.txt"), block_size: 32)) response = client.post(endpoint.path, headers, body) diff --git a/fixtures/async/http/a_protocol.rb b/fixtures/async/http/a_protocol.rb index 0dcde41..5d3870c 100644 --- a/fixtures/async/http/a_protocol.rb +++ b/fixtures/async/http/a_protocol.rb @@ -162,7 +162,7 @@ module HTTP request_received.wait headers.add('etag', 'abcd') - body.close + body.close_write end response = client.post("/", headers, body) @@ -187,7 +187,7 @@ module HTTP response_received.wait headers.add('etag', 'abcd') - body.close + body.close_write end ::Protocol::HTTP::Response[200, headers, body] @@ -395,9 +395,9 @@ module HTTP let(:app) do ::Protocol::HTTP::Middleware.for do |request| Async::HTTP::Body::Hijack.response(request, 200, {}) do |stream| - stream.write content - stream.write content - stream.close + stream.write(content) + stream.write(content) + stream.close_write end end end diff --git a/fixtures/async/http/body/a_writable_body.rb b/fixtures/async/http/body/a_writable_body.rb deleted file mode 100644 index 24c18c3..0000000 --- a/fixtures/async/http/body/a_writable_body.rb +++ /dev/null @@ -1,105 +0,0 @@ -# frozen_string_literal: true - -# Released under the MIT License. -# Copyright, 2019-2023, by Samuel Williams. - -require 'protocol/http/body/deflate' - -module Async - module HTTP - module Body - AWritableBody = Sus::Shared("a writable body") do - it "can write and read data" do - 3.times do |i| - body.write("Hello World #{i}") - expect(body.read).to be == "Hello World #{i}" - end - end - - it "can buffer data in order" do - 3.times do |i| - body.write("Hello World #{i}") - end - - 3.times do |i| - expect(body.read).to be == "Hello World #{i}" - end - end - - with '#join' do - it "can join chunks" do - 3.times do |i| - body.write("#{i}") - end - - body.close - - expect(body.join).to be == "012" - end - end - - with '#each' do - it "can read all data in order" do - 3.times do |i| - body.write("Hello World #{i}") - end - - body.close - - 3.times do |i| - chunk = body.read - expect(chunk).to be == "Hello World #{i}" - end - end - - it "can propagate failures" do - reactor.async do - expect do - body.each do |chunk| - raise RuntimeError.new("It was too big!") - end - end.to raise_exception(RuntimeError, message: be =~ /big/) - end - - expect{ - body.write("Beep boop") # This will cause a failure. - ::Async::Task.current.yield - body.write("Beep boop") # This will fail. - }.to raise_exception(RuntimeError, message: be =~ /big/) - end - - it "can propagate failures in nested bodies" do - nested = ::Protocol::HTTP::Body::Deflate.for(body) - - reactor.async do - expect do - nested.each do |chunk| - raise RuntimeError.new("It was too big!") - end - end.to raise_exception(RuntimeError, message: be =~ /big/) - end - - expect{ - body.write("Beep boop") # This will cause a failure. - ::Async::Task.current.yield - body.write("Beep boop") # This will fail. - }.to raise_exception(RuntimeError, message: be =~ /big/) - end - - it "can consume chunks" do - body.write("Hello World!") - body.close - - expect(body).not.to be(:empty?) - - body.each do |chunk| - expect(chunk).to be == "Hello World!" - end - - expect(body).to be(:empty?) - end - end - end - end - end -end diff --git a/gems.rb b/gems.rb index 4094e8a..52fa32c 100644 --- a/gems.rb +++ b/gems.rb @@ -20,6 +20,8 @@ # gem "protocol-http2", path: "../protocol-http2" # gem "protocol-hpack", path: "../protocol-hpack" +gem "protocol-http", git: "https://github.com/socketry/protocol-http.git" + group :maintenance, optional: true do gem "bake-modernize" gem "bake-gem" diff --git a/lib/async/http/body/delayed.rb b/lib/async/http/body/delayed.rb deleted file mode 100644 index 7b0f57b..0000000 --- a/lib/async/http/body/delayed.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -# Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. -# Copyright, 2020, by Bruno Sutic. -# Copyright, 2023, by Thomas Morgan. - -require 'protocol/http/body/wrapper' - -module Async - module HTTP - module Body - class Delayed < ::Protocol::HTTP::Body::Wrapper - def initialize(body, delay = 0.01) - super(body) - - @delay = delay - end - - def ready? - false - end - - def read - Async::Task.current.sleep(@delay) - - return super - end - end - end - end -end diff --git a/lib/async/http/body/hijack.rb b/lib/async/http/body/hijack.rb index b07fcda..769c45b 100644 --- a/lib/async/http/body/hijack.rb +++ b/lib/async/http/body/hijack.rb @@ -36,7 +36,7 @@ def stream? end def call(stream) - return @block.call(stream) + @block.call(stream) end attr :input diff --git a/lib/async/http/body/pipe.rb b/lib/async/http/body/pipe.rb index 6ef1c0e..93d7a45 100644 --- a/lib/async/http/body/pipe.rb +++ b/lib/async/http/body/pipe.rb @@ -17,7 +17,7 @@ def initialize(input, output = Writable.new, task: Task.current) head, tail = ::Socket.pair(Socket::AF_UNIX, Socket::SOCK_STREAM) - @head = ::IO::Stream::Buffered.new(head) + @head = ::IO::Stream(head) @tail = tail @reader = nil @@ -52,8 +52,10 @@ def reader(task) end @head.close_write + rescue => error + raise ensure - @input.close($!) + @input.close(error) close_head if @writer&.finished? end @@ -68,8 +70,10 @@ def writer(task) while chunk = @head.read_partial @output.write(chunk) end + rescue => error + raise ensure - @output.close($!) + @output.close_write(error) close_head if @reader&.finished? end diff --git a/lib/async/http/body/slowloris.rb b/lib/async/http/body/slowloris.rb deleted file mode 100644 index 6a3d412..0000000 --- a/lib/async/http/body/slowloris.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -# Released under the MIT License. -# Copyright, 2019-2023, by Samuel Williams. - -require_relative 'writable' - -require 'async/clock' - -module Async - module HTTP - module Body - # A dynamic body which you can write to and read from. - class Slowloris < Writable - class ThroughputError < StandardError - def initialize(throughput, minimum_throughput, time_since_last_write) - super("Slow write: #{throughput.round(1)}bytes/s less than required #{minimum_throughput.round}bytes/s.") - end - end - - # In order for this implementation to work correctly, you need to use a LimitedQueue. - # @param minimum_throughput [Integer] the minimum bytes per second otherwise this body will be forcefully closed. - def initialize(*arguments, minimum_throughput: 1024, **options) - super(*arguments, **options) - - @minimum_throughput = minimum_throughput - - @last_write_at = nil - @last_chunk_size = nil - end - - attr :minimum_throughput - - # If #read is called regularly to maintain throughput, that is good. If #read is not called, that is a problem. Throughput is dependent on data being available, from #write, so it doesn't seem particularly problimatic to do this check in #write. - def write(chunk) - if @last_chunk_size - time_since_last_write = Async::Clock.now - @last_write_at - throughput = @last_chunk_size / time_since_last_write - - if throughput < @minimum_throughput - error = ThroughputError.new(throughput, @minimum_throughput, time_since_last_write) - - self.close(error) - end - end - - super.tap do - @last_write_at = Async::Clock.now - @last_chunk_size = chunk&.bytesize - end - end - end - end - end -end diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 19078a6..3463984 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -68,11 +68,23 @@ def each(task: Task.current) stream = write_upgrade_body(protocol) # At this point, the request body is hijacked, so we don't want to call #finish below. - request = nil unless request.body + request = nil response = nil # We must return here as no further request processing can be done: return body.call(stream) + elsif response.status == 101 + # This code path is to support legacy behavior where the response status is set to 101, but the protocol is not upgraded. This may not be a valid use case, but it is supported for compatibility. We expect the response headers to contain the `upgrade` header. + write_response(@version, response.status, response.headers) + + stream = write_tunnel_body(request.version) + + # Same as above: + request = nil + response = nil + + # We must return here as no further request processing can be done: + return body&.call(stream) else write_response(@version, response.status, response.headers) @@ -80,7 +92,7 @@ def each(task: Task.current) stream = write_tunnel_body(request.version) # Same as above: - request = nil unless request.body + request = nil response = nil # We must return here as no further request processing can be done: diff --git a/lib/async/http/protocol/http2/input.rb b/lib/async/http/protocol/http2/input.rb index b29d384..116681b 100644 --- a/lib/async/http/protocol/http2/input.rb +++ b/lib/async/http/protocol/http2/input.rb @@ -3,14 +3,14 @@ # Released under the MIT License. # Copyright, 2020-2023, by Samuel Williams. -require_relative '../../body/writable' +require 'protocol/http/body/writable' module Async module HTTP module Protocol module HTTP2 # A writable body which requests window updates when data is read from it. - class Input < Body::Writable + class Input < ::Protocol::HTTP::Body::Writable def initialize(stream, length) super(length) diff --git a/lib/async/http/protocol/http2/output.rb b/lib/async/http/protocol/http2/output.rb index dee8b1b..4e3d5cf 100644 --- a/lib/async/http/protocol/http2/output.rb +++ b/lib/async/http/protocol/http2/output.rb @@ -50,18 +50,25 @@ def write(chunk) end end - # This method should only be called from within the context of the output task. - def close(error = nil) - if @stream - @stream.finish_output(error) + def close_write(error = nil) + if stream = @stream @stream = nil + stream.finish_output(error) end end + # This method should only be called from within the context of the output task. + def close(error = nil) + close_write(error) + stop(error) + end + # This method should only be called from within the context of the HTTP/2 stream. def stop(error) - @task&.stop - @task = nil + if task = @task + @task = nil + task.stop(error) + end end private @@ -70,10 +77,12 @@ def stream(task) task.annotate("Streaming #{@body} to #{@stream}.") input = @stream.wait_for_input + stream = ::Protocol::HTTP::Body::Stream.new(input, self) - @body.call(::Protocol::HTTP::Body::Stream.new(input, self)) - rescue Async::Stop - # Ignore. + @body.call(stream) + rescue => error + self.close(error) + raise end # Reads chunks from the given body and writes them to the stream as fast as possible. @@ -86,11 +95,17 @@ def passthrough(task) # chunk.clear unless chunk.frozen? # GC.start end - - self.close + rescue => error + raise ensure - @body&.close($!) - @body = nil + # Ensure the body we are reading from is fully closed: + if body = @body + @body = nil + body.close(error) + end + + # Ensure the output of this body is closed: + self.close_write(error) end # Send `maximum_size` bytes of data using the specified `stream`. If the buffer has no more chunks, `END_STREAM` will be sent on the final chunk. diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index 49d2587..60f3fad 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -59,7 +59,7 @@ def process_headers(frame) # TODO this might need to be in an ensure block: if @input and frame.end_stream? - @input.close($!) + @input.close_write @input = nil end rescue ::Protocol::HTTP2::HeaderError => error @@ -98,7 +98,7 @@ def process_data(frame) end if frame.end_stream? - @input.close + @input.close_write @input = nil end end @@ -149,7 +149,7 @@ def closed(error) super if @input - @input.close(error) + @input.close_write(error) @input = nil end diff --git a/test/async/http/body.rb b/test/async/http/body.rb index 37d5adb..1235a54 100644 --- a/test/async/http/body.rb +++ b/test/async/http/body.rb @@ -23,7 +23,7 @@ output.write(chunk.reverse) end - output.close + output.close_write end Protocol::HTTP::Response[200, [], output] @@ -35,7 +35,7 @@ reactor.async do |task| output.write("Hello World!") - output.close + output.close_write end response = client.post("/", {}, output) @@ -58,7 +58,7 @@ notification.wait end - body.close + body.close_write end Protocol::HTTP::Response[200, {}, body] diff --git a/test/async/http/body/hijack.rb b/test/async/http/body/hijack.rb index 73cad02..9d994a2 100644 --- a/test/async/http/body/hijack.rb +++ b/test/async/http/body/hijack.rb @@ -15,7 +15,7 @@ 3.times do stream.write(content) end - stream.close + stream.close_write end end diff --git a/test/async/http/body/pipe.rb b/test/async/http/body/pipe.rb index 0760de2..8e05263 100644 --- a/test/async/http/body/pipe.rb +++ b/test/async/http/body/pipe.rb @@ -20,7 +20,7 @@ include Sus::Fixtures::Async::ReactorContext let(:input_write_duration) {0} - let(:io) { pipe.to_io } + let(:io) {pipe.to_io} def before super @@ -31,14 +31,12 @@ def before input.write("#{first} ") sleep(input_write_duration) if input_write_duration > 0 input.write(second) - input.close + input.close_write end end - def aftrer + after do io.close - - super end it "returns an io socket" do diff --git a/test/async/http/body/slowloris.rb b/test/async/http/body/slowloris.rb deleted file mode 100644 index dc3e48b..0000000 --- a/test/async/http/body/slowloris.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -# Released under the MIT License. -# Copyright, 2019-2023, by Samuel Williams. - -require 'async/http/body/slowloris' - -require 'sus/fixtures/async' -require 'async/http/body/a_writable_body' - -describe Async::HTTP::Body::Slowloris do - include Sus::Fixtures::Async::ReactorContext - - let(:body) {subject.new} - - it_behaves_like Async::HTTP::Body::AWritableBody - - it "closes body with error if throughput is not maintained" do - body.write("Hello World") - - sleep 0.1 - - expect do - body.write("Hello World") - end.to raise_exception(Async::HTTP::Body::Slowloris::ThroughputError, message: be =~ /Slow write/) - end - - it "doesn't close body if throughput is exceeded" do - body.write("Hello World") - - expect do - body.write("Hello World") - end.not.to raise_exception - end -end diff --git a/test/async/http/body/writable.rb b/test/async/http/body/writable.rb deleted file mode 100644 index 9d553a5..0000000 --- a/test/async/http/body/writable.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -# Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. - -require 'async/http/body/slowloris' - -require 'sus/fixtures/async' -require 'async/http/body/a_writable_body' - -describe Async::HTTP::Body::Writable do - include Sus::Fixtures::Async::ReactorContext - - let(:body) {subject.new} - - it_behaves_like Async::HTTP::Body::AWritableBody -end diff --git a/test/async/http/proxy.rb b/test/async/http/proxy.rb index b3b4708..7b37ba3 100644 --- a/test/async/http/proxy.rb +++ b/test/async/http/proxy.rb @@ -57,7 +57,7 @@ expect(response).to be(:success?) input.write(data) - input.close + input.close_write expect(response.read).to be == data end @@ -74,7 +74,7 @@ stream.flush end - stream.close + stream.close_write end end end From 6b3066a62e495f48a1a1652240f1dc4af7ac7115 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 11 Sep 2024 10:39:00 +1200 Subject: [PATCH 069/125] Bump dependencies. --- async-http.gemspec | 2 +- gems.rb | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index c3a4df0..e39e376 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.7" spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" - spec.add_dependency "protocol-http", "~> 0.33" + spec.add_dependency "protocol-http", "~> 0.34" spec.add_dependency "protocol-http1", "~> 0.20" spec.add_dependency "protocol-http2", "~> 0.18" spec.add_dependency "traces", ">= 0.10" diff --git a/gems.rb b/gems.rb index 52fa32c..4094e8a 100644 --- a/gems.rb +++ b/gems.rb @@ -20,8 +20,6 @@ # gem "protocol-http2", path: "../protocol-http2" # gem "protocol-hpack", path: "../protocol-hpack" -gem "protocol-http", git: "https://github.com/socketry/protocol-http.git" - group :maintenance, optional: true do gem "bake-modernize" gem "bake-gem" From 489c468dbae5e137c45fa5fc65646d2db3a76e81 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 11 Sep 2024 11:29:38 +1200 Subject: [PATCH 070/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 4b9a3bd..e912314 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.75.0" + VERSION = "0.76.0" end end From 1c4b5ab125d9ac22148cece4678c41f1e582bac7 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 13 Sep 2024 14:57:33 +1200 Subject: [PATCH 071/125] Slightly improved handling of version in HTTP/1 server. --- lib/async/http/protocol/http1/server.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 3463984..51d32ca 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -47,6 +47,7 @@ def each(task: Task.current) while request = next_request response = yield(request, self) + version = request.version body = response&.body if hijacked? @@ -77,7 +78,7 @@ def each(task: Task.current) # This code path is to support legacy behavior where the response status is set to 101, but the protocol is not upgraded. This may not be a valid use case, but it is supported for compatibility. We expect the response headers to contain the `upgrade` header. write_response(@version, response.status, response.headers) - stream = write_tunnel_body(request.version) + stream = write_tunnel_body(version) # Same as above: request = nil @@ -89,7 +90,7 @@ def each(task: Task.current) write_response(@version, response.status, response.headers) if request.connect? and response.success? - stream = write_tunnel_body(request.version) + stream = write_tunnel_body(version) # Same as above: request = nil @@ -99,7 +100,6 @@ def each(task: Task.current) return body.call(stream) else head = request.head? - version = request.version # Same as above: request = nil unless request.body @@ -114,7 +114,7 @@ def each(task: Task.current) else # If the request failed to generate a response, it was an internal server error: write_response(@version, 500, {}) - write_body(request.version, nil) + write_body(version, nil) end # Gracefully finish reading the request body if it was not already done so. From 2f3180acad2a5b1362c02bc1f98f66e8382eafa7 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 17 Sep 2024 10:08:24 +1200 Subject: [PATCH 072/125] Streaming tests. (#182) * Wait for input to be consumed before continuing. * Update dependencies. --- async-http.gemspec | 2 +- lib/async/http/body/finishable.rb | 39 +++++ lib/async/http/client.rb | 3 + lib/async/http/protocol/http1/server.rb | 16 +- .../http/middleware/location_redirector.rb | 2 + test/protocol/http/body/stream.rb | 142 ++++++++++++++++++ test/protocol/http/body/streamable.rb | 134 +++++++++++++++++ 7 files changed, 333 insertions(+), 5 deletions(-) create mode 100644 lib/async/http/body/finishable.rb create mode 100644 test/protocol/http/body/stream.rb create mode 100644 test/protocol/http/body/streamable.rb diff --git a/async-http.gemspec b/async-http.gemspec index e39e376..c316537 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.7" spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" - spec.add_dependency "protocol-http", "~> 0.34" + spec.add_dependency "protocol-http", "~> 0.35" spec.add_dependency "protocol-http1", "~> 0.20" spec.add_dependency "protocol-http2", "~> 0.18" spec.add_dependency "traces", ">= 0.10" diff --git a/lib/async/http/body/finishable.rb b/lib/async/http/body/finishable.rb new file mode 100644 index 0000000..cfdf996 --- /dev/null +++ b/lib/async/http/body/finishable.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2019-2023, by Samuel Williams. + +require 'protocol/http/body/wrapper' +require 'async/variable' + +module Async + module HTTP + module Body + class Finishable < ::Protocol::HTTP::Body::Wrapper + def initialize(body) + super(body) + + @closed = Async::Variable.new + @error = nil + end + + def close(error = nil) + unless @closed.resolved? + @error = error + @closed.value = true + end + + super + end + + def wait + @closed.wait + end + + def inspect + "#<#{self.class} closed=#{@closed} error=#{@error}> | #{super}" + end + end + end + end +end diff --git a/lib/async/http/client.rb b/lib/async/http/client.rb index bbaeb18..53468a7 100755 --- a/lib/async/http/client.rb +++ b/lib/async/http/client.rb @@ -188,6 +188,9 @@ def make_response(request, connection) # The connection won't be released until the body is completely read/released. ::Protocol::HTTP::Body::Completable.wrap(response) do + # TODO: We should probably wait until the request is fully consumed and/or the connection is ready before releasing it back into the pool. + + # Release the connection back into the pool: @pool.release(connection) end diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 51d32ca..2451363 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -7,6 +7,8 @@ # Copyright, 2024, by Anton Zhuravsky. require_relative 'connection' +require_relative '../../body/finishable' + require 'console/event/failure' module Async @@ -46,6 +48,11 @@ def each(task: Task.current) task.annotate("Reading #{self.version} requests for #{self.class}.") while request = next_request + if body = request.body + finishable = Body::Finishable.new(body) + request.body = finishable + end + response = yield(request, self) version = request.version body = response&.body @@ -102,23 +109,24 @@ def each(task: Task.current) head = request.head? # Same as above: - request = nil unless request.body + request = nil response = nil write_body(version, body, head, trailer) end end - # We are done with the body, you shouldn't need to call close on it: + # We are done with the body: body = nil else # If the request failed to generate a response, it was an internal server error: write_response(@version, 500, {}) write_body(version, nil) + + request&.finish end - # Gracefully finish reading the request body if it was not already done so. - request&.each{} + finishable&.wait # This ensures we yield at least once every iteration of the loop and allow other fibers to execute. task.yield diff --git a/test/async/http/middleware/location_redirector.rb b/test/async/http/middleware/location_redirector.rb index 1efb8ee..9a950b9 100644 --- a/test/async/http/middleware/location_redirector.rb +++ b/test/async/http/middleware/location_redirector.rb @@ -18,6 +18,8 @@ with '301' do let(:app) do Protocol::HTTP::Middleware.for do |request| + request.finish # TODO: request.discard - or some default handling? + case request.path when '/home' Protocol::HTTP::Response[301, {'location' => '/'}, []] diff --git a/test/protocol/http/body/stream.rb b/test/protocol/http/body/stream.rb new file mode 100644 index 0000000..430a1ee --- /dev/null +++ b/test/protocol/http/body/stream.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2024, by Samuel Williams. + +require "async/http/protocol/http" +require "protocol/http/body/streamable" +require "sus/fixtures/async/http" + +AnEchoServer = Sus::Shared("an echo server") do + let(:app) do + ::Protocol::HTTP::Middleware.for do |request| + output = ::Protocol::HTTP::Body::Writable.new + + Async do + stream = ::Protocol::HTTP::Body::Stream.new(request.body, output) + + Console.debug(self, "Echoing chunks...") + while chunk = stream.readpartial(1024) + Console.debug(self, "Reading chunk:", chunk: chunk) + stream.write(chunk) + end + rescue EOFError + Console.debug(self, "EOF.") + # Ignore. + ensure + Console.debug(self, "Closing stream.") + stream.close + end + + ::Protocol::HTTP::Response[200, {}, output] + end + end + + it "should echo the request body" do + chunks = ["Hello,", "World!"] + response_chunks = Queue.new + + output = ::Protocol::HTTP::Body::Writable.new + response = client.post("/", body: output) + stream = ::Protocol::HTTP::Body::Stream.new(response.body, output) + + begin + Console.debug(self, "Echoing chunks...") + chunks.each do |chunk| + Console.debug(self, "Writing chunk:", chunk: chunk) + stream.write(chunk) + end + + Console.debug(self, "Closing write.") + stream.close_write + + Console.debug(self, "Reading chunks...") + while chunk = stream.readpartial(1024) + Console.debug(self, "Reading chunk:", chunk: chunk) + response_chunks << chunk + end + rescue EOFError + Console.debug(self, "EOF.") + # Ignore. + ensure + Console.debug(self, "Closing stream.") + stream.close + response_chunks.close + end + + chunks.each do |chunk| + expect(response_chunks.pop).to be == chunk + end + end +end + +AnEchoClient = Sus::Shared("an echo client") do + let(:chunks) {["Hello,", "World!"]} + let(:response_chunks) {Queue.new} + + let(:app) do + ::Protocol::HTTP::Middleware.for do |request| + output = ::Protocol::HTTP::Body::Writable.new + + Async do + stream = ::Protocol::HTTP::Body::Stream.new(request.body, output) + + Console.debug(self, "Echoing chunks...") + chunks.each do |chunk| + stream.write(chunk) + end + + Console.debug(self, "Closing write.") + stream.close_write + + Console.debug(self, "Reading chunks...") + while chunk = stream.readpartial(1024) + Console.debug(self, "Reading chunk:", chunk: chunk) + response_chunks << chunk + end + rescue EOFError + Console.debug(self, "EOF.") + # Ignore. + ensure + Console.debug(self, "Closing stream.") + stream.close + end + + ::Protocol::HTTP::Response[200, {}, output] + end + end + + it "should echo the response body" do + output = ::Protocol::HTTP::Body::Writable.new + response = client.post("/", body: output) + stream = ::Protocol::HTTP::Body::Stream.new(response.body, output) + + begin + Console.debug(self, "Echoing chunks...") + while chunk = stream.readpartial(1024) + stream.write(chunk) + end + rescue EOFError + Console.debug(self, "EOF.") + # Ignore. + ensure + Console.debug(self, "Closing stream.") + stream.close + end + + chunks.each do |chunk| + expect(response_chunks.pop).to be == chunk + end + end +end + +[Async::HTTP::Protocol::HTTP1, Async::HTTP::Protocol::HTTP2].each do |protocol| + describe protocol, unique: protocol.name do + include Sus::Fixtures::Async::HTTP::ServerContext + + let(:protocol) {subject} + + it_behaves_like AnEchoServer + it_behaves_like AnEchoClient + end +end diff --git a/test/protocol/http/body/streamable.rb b/test/protocol/http/body/streamable.rb new file mode 100644 index 0000000..17f2cef --- /dev/null +++ b/test/protocol/http/body/streamable.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2024, by Samuel Williams. + +require "async/http/protocol/http" +require "protocol/http/body/streamable" +require "sus/fixtures/async/http" + +AnEchoServer = Sus::Shared("an echo server") do + let(:app) do + ::Protocol::HTTP::Middleware.for do |request| + streamable = ::Protocol::HTTP::Body::Streamable.response(request) do |stream| + Console.debug(self, "Echoing chunks...") + while chunk = stream.readpartial(1024) + Console.debug(self, "Reading chunk:", chunk: chunk) + stream.write(chunk) + end + rescue EOFError + Console.debug(self, "EOF.") + # Ignore. + ensure + Console.debug(self, "Closing stream.") + stream.close + end + + ::Protocol::HTTP::Response[200, {}, streamable] + end + end + + it "should echo the request body" do + chunks = ["Hello,", "World!"] + response_chunks = Queue.new + + output = ::Protocol::HTTP::Body::Writable.new + response = client.post("/", body: output) + stream = ::Protocol::HTTP::Body::Stream.new(response.body, output) + + begin + Console.debug(self, "Echoing chunks...") + chunks.each do |chunk| + Console.debug(self, "Writing chunk:", chunk: chunk) + stream.write(chunk) + end + + Console.debug(self, "Closing write.") + stream.close_write + + Console.debug(self, "Reading chunks...") + while chunk = stream.readpartial(1024) + Console.debug(self, "Reading chunk:", chunk: chunk) + response_chunks << chunk + end + rescue EOFError + Console.debug(self, "EOF.") + # Ignore. + ensure + Console.debug(self, "Closing stream.") + stream.close + response_chunks.close + end + + chunks.each do |chunk| + expect(response_chunks.pop).to be == chunk + end + end +end + +AnEchoClient = Sus::Shared("an echo client") do + let(:chunks) {["Hello,", "World!"]} + let(:response_chunks) {Queue.new} + + let(:app) do + ::Protocol::HTTP::Middleware.for do |request| + streamable = ::Protocol::HTTP::Body::Streamable.response(request) do |stream| + Console.debug(self, "Echoing chunks...") + chunks.each do |chunk| + stream.write(chunk) + end + + Console.debug(self, "Closing write.") + stream.close_write + + Console.debug(self, "Reading chunks...") + while chunk = stream.readpartial(1024) + Console.debug(self, "Reading chunk:", chunk: chunk) + response_chunks << chunk + end + rescue EOFError + Console.debug(self, "EOF.") + # Ignore. + ensure + Console.debug(self, "Closing stream.") + stream.close + end + + ::Protocol::HTTP::Response[200, {}, streamable] + end + end + + it "should echo the response body" do + output = ::Protocol::HTTP::Body::Writable.new + response = client.post("/", body: output) + stream = ::Protocol::HTTP::Body::Stream.new(response.body, output) + + begin + Console.debug(self, "Echoing chunks...") + while chunk = stream.readpartial(1024) + stream.write(chunk) + end + rescue EOFError + Console.debug(self, "EOF.") + # Ignore. + ensure + Console.debug(self, "Closing stream.") + stream.close + end + + chunks.each do |chunk| + expect(response_chunks.pop).to be == chunk + end + end +end + +[Async::HTTP::Protocol::HTTP1, Async::HTTP::Protocol::HTTP2].each do |protocol| + describe protocol, unique: protocol.name do + include Sus::Fixtures::Async::HTTP::ServerContext + + let(:protocol) {subject} + + it_behaves_like AnEchoServer + it_behaves_like AnEchoClient + end +end From 43940cadaeddb7870473a39566afb1b902751aa8 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 19 Sep 2024 23:54:09 +1200 Subject: [PATCH 073/125] Stateful HTTP/1 connection handling. --- async-http.gemspec | 4 ++-- fixtures/async/http/a_protocol.rb | 5 ++-- lib/async/http/body/finishable.rb | 19 ++++++++++++++- lib/async/http/client.rb | 11 +++------ lib/async/http/protocol/http1/client.rb | 23 +++++++++++++++---- lib/async/http/protocol/http1/connection.rb | 7 +++--- lib/async/http/protocol/http1/response.rb | 8 +++++++ lib/async/http/protocol/http1/server.rb | 4 ++-- lib/async/http/protocol/http2/response.rb | 10 ++++++++ lib/async/http/protocol/http2/stream.rb | 8 +++++++ lib/async/http/proxy.rb | 1 + .../http/middleware/location_redirector.rb | 2 -- test/async/http/protocol/http11.rb | 4 ++-- test/async/http/proxy.rb | 2 ++ 14 files changed, 80 insertions(+), 28 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index c316537..8df2f8c 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,8 +28,8 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.7" spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" - spec.add_dependency "protocol-http", "~> 0.35" - spec.add_dependency "protocol-http1", "~> 0.20" + spec.add_dependency "protocol-http", "~> 0.37" + spec.add_dependency "protocol-http1", "~> 0.25" spec.add_dependency "protocol-http2", "~> 0.18" spec.add_dependency "traces", ">= 0.10" end diff --git a/fixtures/async/http/a_protocol.rb b/fixtures/async/http/a_protocol.rb index 5d3870c..5248333 100644 --- a/fixtures/async/http/a_protocol.rb +++ b/fixtures/async/http/a_protocol.rb @@ -14,6 +14,7 @@ require 'tempfile' require 'protocol/http/body/file' +require "protocol/http/body/buffered" require 'sus/fixtures/async/http' @@ -100,7 +101,7 @@ module HTTP end with 'buffered body' do - let(:body) {Async::HTTP::Body::Buffered.new(["Hello World"])} + let(:body) {::Protocol::HTTP::Body::Buffered.new(["Hello World"])} let(:response) {::Protocol::HTTP::Response[200, {}, body]} let(:app) do @@ -410,7 +411,7 @@ module HTTP end with 'body with incorrect length' do - let(:bad_body) {Async::HTTP::Body::Buffered.new(["Borked"], 10)} + let(:bad_body) {::Protocol::HTTP::Body::Buffered.new(["Borked"], 10)} let(:app) do ::Protocol::HTTP::Middleware.for do |request| diff --git a/lib/async/http/body/finishable.rb b/lib/async/http/body/finishable.rb index cfdf996..4dcfe03 100644 --- a/lib/async/http/body/finishable.rb +++ b/lib/async/http/body/finishable.rb @@ -9,12 +9,25 @@ module Async module HTTP module Body + # Keeps track of whether a body is being read, and if so, waits for it to be closed. class Finishable < ::Protocol::HTTP::Body::Wrapper def initialize(body) super(body) @closed = Async::Variable.new @error = nil + + @reading = false + end + + def reading? + @reading + end + + def read + @reading = true + + super end def close(error = nil) @@ -27,7 +40,11 @@ def close(error = nil) end def wait - @closed.wait + if @reading + @closed.wait + else + self.discard + end end def inspect diff --git a/lib/async/http/client.rb b/lib/async/http/client.rb index 53468a7..7c2c8a8 100755 --- a/lib/async/http/client.rb +++ b/lib/async/http/client.rb @@ -14,6 +14,7 @@ require 'traces/provider' require_relative 'protocol' +require_relative 'body/finishable' module Async module HTTP @@ -140,7 +141,7 @@ def call(request) def inspect "#<#{self.class} authority=#{@authority.inspect}>" end - + Traces::Provider(self) do def call(request) attributes = { @@ -186,13 +187,7 @@ def call(request) def make_response(request, connection) response = request.call(connection) - # The connection won't be released until the body is completely read/released. - ::Protocol::HTTP::Body::Completable.wrap(response) do - # TODO: We should probably wait until the request is fully consumed and/or the connection is ready before releasing it back into the pool. - - # Release the connection back into the pool: - @pool.release(connection) - end + response.pool = @pool return response end diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index c7fa99b..c34eef0 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -10,11 +10,25 @@ module HTTP module Protocol module HTTP1 class Client < Connection + def initialize(...) + super + + @pool = nil + end + + attr_accessor :pool + + def closed! + super + + if pool = @pool + @pool = nil + pool.release(self) + end + end + # Used by the client to send requests to the remote server. def call(request, task: Task.current) - # We need to keep track of connections which are not in the initial "ready" state. - @ready = false - Console.logger.debug(self) {"#{request.method} #{request.path} #{request.headers.inspect}"} # Mark the start of the trailers: @@ -54,12 +68,11 @@ def call(request, task: Task.current) end response = Response.read(self, request) - @ready = true return response rescue # This will ensure that #reusable? returns false. - @stream.close + self.close raise end diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index 23a3ed6..0f88c15 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -16,12 +16,11 @@ class Connection < ::Protocol::HTTP1::Connection def initialize(stream, version) super(stream) - @ready = true @version = version end def to_s - "\#<#{self.class} negotiated #{@version}, currently #{@ready ? 'ready' : 'in-use'}>" + "\#<#{self.class} negotiated #{@version}, #{@state}>" end def as_json(...) @@ -62,11 +61,11 @@ def concurrency # Can we use this connection to make requests? def viable? - @ready && @stream&.readable? + self.idle? && @stream&.readable? end def reusable? - @ready && @persistent && @stream && !@stream.closed? + @persistent && @stream && !@stream.closed? end end end diff --git a/lib/async/http/protocol/http1/response.rb b/lib/async/http/protocol/http1/response.rb index a7a8c6f..08afc35 100644 --- a/lib/async/http/protocol/http1/response.rb +++ b/lib/async/http/protocol/http1/response.rb @@ -39,6 +39,14 @@ def initialize(connection, version, status, reason, headers, body) super(version, status, headers, body, protocol) end + def pool=(pool) + if @connection.idle? or @connection.closed? + pool.release(@connection) + else + @connection.pool = pool + end + end + def connection @connection end diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 2451363..b30e0c6 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -22,7 +22,7 @@ def fail_request(status) write_body(@version, nil) rescue => error # At this point, there is very little we can do to recover: - Console::Event::Failure.for(error).emit(self, "Failed to write failure response.", severity: :debug) + Console::Event::Failure.for(error).emit(self, "Failed to write failure response!", severity: :debug) end def next_request @@ -37,7 +37,7 @@ def next_request end return request - rescue ::Protocol::HTTP1::BadRequest + rescue ::Protocol::HTTP1::BadRequest => error fail_request(400) # Conceivably we could retry here, but we don't really know how bad the error is, so it's better to just fail: raise diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index 5d79713..923b5c0 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -137,6 +137,16 @@ def initialize(stream) attr :stream attr :request + def pool=(pool) + # If we are already closed, the stream can be released now: + if @stream.closed? + pool.release(@stream.connection) + else + # Otherwise, we will release the stream when it is closed: + @stream.pool = pool + end + end + def connection @stream.connection end diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index 60f3fad..c9a98ee 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -20,6 +20,8 @@ def initialize(*) @headers = nil + @pool = nil + # Input buffer, reading request body, or response body (receive_data): @length = nil @input = nil @@ -30,6 +32,8 @@ def initialize(*) attr_accessor :headers + attr_accessor :pool + attr :input def add_header(key, value) @@ -158,6 +162,10 @@ def closed(error) @output = nil end + if pool = @pool and @connection + pool.release(@connection) + end + return self end end diff --git a/lib/async/http/proxy.rb b/lib/async/http/proxy.rb index 338031b..d1d7bee 100644 --- a/lib/async/http/proxy.rb +++ b/lib/async/http/proxy.rb @@ -96,6 +96,7 @@ def connect(&block) end else # This ensures we don't leave a response dangling: + input.close response.close raise ConnectFailure, response diff --git a/test/async/http/middleware/location_redirector.rb b/test/async/http/middleware/location_redirector.rb index 9a950b9..1efb8ee 100644 --- a/test/async/http/middleware/location_redirector.rb +++ b/test/async/http/middleware/location_redirector.rb @@ -18,8 +18,6 @@ with '301' do let(:app) do Protocol::HTTP::Middleware.for do |request| - request.finish # TODO: request.discard - or some default handling? - case request.path when '/home' Protocol::HTTP::Response[301, {'location' => '/'}, []] diff --git a/test/async/http/protocol/http11.rb b/test/async/http/protocol/http11.rb index 3fa8db7..c8bec4b 100755 --- a/test/async/http/protocol/http11.rb +++ b/test/async/http/protocol/http11.rb @@ -21,7 +21,7 @@ response = client.get("/") connection = response.connection - expect(connection.as_json).to be == "#" + expect(connection.as_json).to be =~ /Async::HTTP::Protocol::HTTP1::Client negotiated HTTP/ ensure response&.close end @@ -109,7 +109,7 @@ def around end with 'full hijack with empty response' do - let(:body) {Async::HTTP::Body::Buffered.new([], 0)} + let(:body) {::Protocol::HTTP::Body::Buffered.new([], 0)} let(:app) do ::Protocol::HTTP::Middleware.for do |request| diff --git a/test/async/http/proxy.rb b/test/async/http/proxy.rb index 7b37ba3..0d493a3 100644 --- a/test/async/http/proxy.rb +++ b/test/async/http/proxy.rb @@ -42,6 +42,8 @@ stream.close_read stream.write(chunk) + stream.close_write + ensure stream.close end end From 6b7ae6896d907f99fe9c06cde09023c78036a16b Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 20 Sep 2024 10:32:14 +1200 Subject: [PATCH 074/125] Modernize gem. --- .github/workflows/test-coverage.yaml | 2 + .rubocop.yml | 4 + async-http.gemspec | 6 +- bake.rb | 4 +- bake/async/http.rb | 6 +- bake/async/http/h2spec.rb | 12 +-- config/sus.rb | 10 +- examples/compare/benchmark.rb | 14 +-- examples/download/chunked.rb | 18 ++-- examples/fetch/config.ru | 4 +- examples/fetch/gems.rb | 6 +- examples/google/codeotaku.rb | 2 +- examples/google/multiple.rb | 10 +- examples/google/search.rb | 6 +- examples/header-lowercase/benchmark.rb | 24 ++--- examples/hello/config.ru | 8 +- examples/licenses/gemspect.rb | 22 ++-- examples/licenses/list.rb | 22 ++-- examples/race/client.rb | 16 +-- examples/race/server.rb | 12 +-- examples/request.rb | 10 +- examples/request/http10.rb | 8 +- examples/stream/stop.rb | 6 +- examples/trenni/Gemfile | 2 +- examples/trenni/streaming.rb | 10 +- examples/upload/client.rb | 12 +-- examples/upload/server.rb | 12 +-- examples/upload/upload.rb | 10 +- fixtures/async/http/a_protocol.rb | 102 +++++++++--------- gems.rb | 2 +- lib/async/http.rb | 10 +- lib/async/http/body.rb | 6 +- lib/async/http/body/finishable.rb | 6 +- lib/async/http/body/hijack.rb | 8 +- lib/async/http/body/pipe.rb | 2 +- lib/async/http/body/writable.rb | 6 +- lib/async/http/client.rb | 28 ++--- lib/async/http/endpoint.rb | 20 ++-- lib/async/http/internet.rb | 10 +- lib/async/http/internet/instance.rb | 2 +- .../http/middleware/location_redirector.rb | 16 +-- lib/async/http/mock.rb | 2 +- lib/async/http/mock/endpoint.rb | 4 +- lib/async/http/protocol.rb | 6 +- lib/async/http/protocol/http.rb | 4 +- lib/async/http/protocol/http1.rb | 6 +- lib/async/http/protocol/http1/client.rb | 2 +- lib/async/http/protocol/http1/connection.rb | 6 +- lib/async/http/protocol/http1/request.rb | 6 +- lib/async/http/protocol/http1/response.rb | 4 +- lib/async/http/protocol/http1/server.rb | 6 +- lib/async/http/protocol/http10.rb | 2 +- lib/async/http/protocol/http11.rb | 2 +- lib/async/http/protocol/http2.rb | 6 +- lib/async/http/protocol/http2/client.rb | 8 +- lib/async/http/protocol/http2/connection.rb | 24 ++--- lib/async/http/protocol/http2/input.rb | 4 +- lib/async/http/protocol/http2/output.rb | 4 +- lib/async/http/protocol/http2/request.rb | 8 +- lib/async/http/protocol/http2/response.rb | 8 +- lib/async/http/protocol/http2/server.rb | 6 +- lib/async/http/protocol/http2/stream.rb | 8 +- lib/async/http/protocol/https.rb | 6 +- lib/async/http/protocol/request.rb | 6 +- lib/async/http/protocol/response.rb | 6 +- lib/async/http/proxy.rb | 6 +- lib/async/http/reference.rb | 4 +- lib/async/http/relative_location.rb | 2 +- lib/async/http/server.rb | 26 ++--- lib/async/http/statistics.rb | 8 +- test/async/http/body.rb | 14 +-- test/async/http/body/hijack.rb | 8 +- test/async/http/body/pipe.rb | 26 ++--- test/async/http/client.rb | 28 ++--- test/async/http/client/google.rb | 18 ++-- test/async/http/endpoint.rb | 42 ++++---- test/async/http/internet.rb | 14 +-- test/async/http/internet/instance.rb | 6 +- .../http/middleware/location_redirector.rb | 72 ++++++------- test/async/http/mock.rb | 10 +- test/async/http/protocol/http.rb | 22 ++-- test/async/http/protocol/http10.rb | 6 +- test/async/http/protocol/http11.rb | 16 +-- test/async/http/protocol/http11/desync.rb | 8 +- test/async/http/protocol/http2.rb | 26 ++--- test/async/http/proxy.rb | 42 ++++---- test/async/http/retry.rb | 12 +-- test/async/http/server.rb | 10 +- test/async/http/ssl.rb | 12 +-- test/async/http/statistics.rb | 6 +- test/rack/test.rb | 8 +- 91 files changed, 549 insertions(+), 543 deletions(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index ffa0927..50e9293 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -36,6 +36,8 @@ jobs: - uses: actions/upload-artifact@v4 with: + include-hidden-files: true + if-no-files-found: error name: coverage-${{matrix.os}}-${{matrix.ruby}} path: .covered.db diff --git a/.rubocop.yml b/.rubocop.yml index a2447c2..3b8d476 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -47,3 +47,7 @@ Layout/EmptyLinesAroundModuleBody: Style/FrozenStringLiteralComment: Enabled: true + +Style/StringLiterals: + Enabled: true + EnforcedStyle: double_quotes diff --git a/async-http.gemspec b/async-http.gemspec index 8df2f8c..af9026d 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -10,8 +10,8 @@ Gem::Specification.new do |spec| spec.authors = ["Samuel Williams", "Brian Morearty", "Bruno Sutic", "Janko Marohnić", "Thomas Morgan", "Adam Daniels", "Igor Sidorov", "Anton Zhuravsky", "Cyril Roelandt", "Denis Talakevich", "Hal Brodigan", "Ian Ker-Seymer", "Josh Huber", "Marco Concetto Rudilosso", "Olle Jonsson", "Orgad Shaneh", "Sam Shadwell", "Stefan Wrobel", "Tim Meusel", "Trevor Turk", "Viacheslav Koval", "dependabot[bot]"] spec.license = "MIT" - spec.cert_chain = ['release.cert'] - spec.signing_key = File.expand_path('~/.gem/release.pem') + spec.cert_chain = ["release.cert"] + spec.signing_key = File.expand_path("~/.gem/release.pem") spec.homepage = "https://github.com/socketry/async-http" @@ -20,7 +20,7 @@ Gem::Specification.new do |spec| "source_code_uri" => "https://github.com/socketry/async-http.git", } - spec.files = Dir.glob(['{bake,lib}/**/*', '*.md'], File::FNM_DOTMATCH, base: __dir__) + spec.files = Dir.glob(["{bake,lib}/**/*", "*.md"], File::FNM_DOTMATCH, base: __dir__) spec.required_ruby_version = ">= 3.1" diff --git a/bake.rb b/bake.rb index 6971885..60987a2 100644 --- a/bake.rb +++ b/bake.rb @@ -7,6 +7,6 @@ # # @parameter version [String] The new version number. def after_gem_release_version_increment(version) - context['releases:update'].call(version) - context['utopia:project:readme:update'].call + context["releases:update"].call(version) + context["utopia:project:readme:update"].call end diff --git a/bake/async/http.rb b/bake/async/http.rb index a8e721b..ab7569b 100644 --- a/bake/async/http.rb +++ b/bake/async/http.rb @@ -1,14 +1,14 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. # Fetch the specified URL and print the response. # @param url [String] the URL to parse and fetch. # @param method [String] the HTTP method to use. def fetch(url, method:) - require 'async/http/internet' - require 'kernel/sync' + require "async/http/internet" + require "kernel/sync" terminal = Console::Terminal.for($stdout) terminal[:request] = terminal.style(:blue, nil, :bold) diff --git a/bake/async/http/h2spec.rb b/bake/async/http/h2spec.rb index 7807c85..960e8b3 100644 --- a/bake/async/http/h2spec.rb +++ b/bake/async/http/h2spec.rb @@ -21,12 +21,12 @@ def test private def server - require 'async' - require 'async/container' - require 'async/http/server' - require 'io/endpoint/host_endpoint' + require "async" + require "async/container" + require "async/http/server" + require "io/endpoint/host_endpoint" - endpoint = IO::Endpoint.tcp('127.0.0.1', 7272) + endpoint = IO::Endpoint.tcp("127.0.0.1", 7272) container = Async::Container.new @@ -34,7 +34,7 @@ def server container.run(count: 1) do server = Async::HTTP::Server.for(endpoint, protocol: Async::HTTP::Protocol::HTTP2, scheme: "https") do |request| - Protocol::HTTP::Response[200, {'content-type' => 'text/plain'}, ["Hello World"]] + Protocol::HTTP::Response[200, {"content-type" => "text/plain"}, ["Hello World"]] end Async do diff --git a/config/sus.rb b/config/sus.rb index ee30cfc..13414a2 100644 --- a/config/sus.rb +++ b/config/sus.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2023, by Samuel Williams. +# Copyright, 2017-2024, by Samuel Williams. # Copyright, 2018, by Janko Marohnić. -ENV['CONSOLE_LEVEL'] ||= 'fatal' +ENV["CONSOLE_LEVEL"] ||= "fatal" -require 'covered/sus' +require "covered/sus" include Covered::Sus -require 'traces' -ENV['TRACES_BACKEND'] ||= 'traces/backend/test' +require "traces" +ENV["TRACES_BACKEND"] ||= "traces/backend/test" diff --git a/examples/compare/benchmark.rb b/examples/compare/benchmark.rb index 7130090..ba24522 100755 --- a/examples/compare/benchmark.rb +++ b/examples/compare/benchmark.rb @@ -2,16 +2,16 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. -require 'benchmark' +require "benchmark" -require 'httpx' +require "httpx" -require 'async' -require 'async/barrier' -require 'async/semaphore' -require 'async/http/internet' +require "async" +require "async/barrier" +require "async/semaphore" +require "async/http/internet" URL = "https://www.codeotaku.com/index" REPEATS = 10 diff --git a/examples/download/chunked.rb b/examples/download/chunked.rb index 5953fab..f2b3410 100755 --- a/examples/download/chunked.rb +++ b/examples/download/chunked.rb @@ -2,14 +2,14 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. -require 'async' -require 'async/clock' -require 'async/barrier' -require 'async/semaphore' -require_relative '../../lib/async/http/endpoint' -require_relative '../../lib/async/http/client' +require "async" +require "async/clock" +require "async/barrier" +require "async/semaphore" +require_relative "../../lib/async/http/endpoint" +require_relative "../../lib/async/http/client" Async do url = "https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv" @@ -17,7 +17,7 @@ endpoint = Async::HTTP::Endpoint.parse(url) client = Async::HTTP::Client.new(endpoint) - headers = {'user-agent' => 'curl/7.69.1', 'accept' => '*/*'} + headers = {"user-agent" => "curl/7.69.1", "accept" => "*/*"} file = File.open("products.csv", "w") Console.logger.info(self) {"Saving download to #{Dir.pwd}"} @@ -27,7 +27,7 @@ content_length = nil if response.success? - unless response.headers['accept-ranges'].include?('bytes') + unless response.headers["accept-ranges"].include?("bytes") raise "Does not advertise support for accept-ranges: bytes!" end diff --git a/examples/fetch/config.ru b/examples/fetch/config.ru index 0a29fa6..1410bee 100644 --- a/examples/fetch/config.ru +++ b/examples/fetch/config.ru @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'rack' +require "rack" class Echo def initialize(app) @@ -24,6 +24,6 @@ end use Echo -use Rack::Static, :urls => [''], :root => 'public', :index => 'index.html' +use Rack::Static, :urls => [""], :root => "public", :index => "index.html" run lambda{|env| [404, {}, []]} diff --git a/examples/fetch/gems.rb b/examples/fetch/gems.rb index a4ca0a5..38fb27e 100644 --- a/examples/fetch/gems.rb +++ b/examples/fetch/gems.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. -gem 'rack' -gem 'falcon' +gem "rack" +gem "falcon" diff --git a/examples/google/codeotaku.rb b/examples/google/codeotaku.rb index fcffc79..c2834c5 100755 --- a/examples/google/codeotaku.rb +++ b/examples/google/codeotaku.rb @@ -12,7 +12,7 @@ URL = "https://www.codeotaku.com/index" ENDPOINT = Async::HTTP::Endpoint.parse(URL) -if count = ENV['COUNT']&.to_i +if count = ENV["COUNT"]&.to_i terms = terms.first(count) end diff --git a/examples/google/multiple.rb b/examples/google/multiple.rb index ad9123f..ca83ad0 100755 --- a/examples/google/multiple.rb +++ b/examples/google/multiple.rb @@ -2,12 +2,12 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2023, by Samuel Williams. +# Copyright, 2023-2024, by Samuel Williams. -require 'async' -require 'async/barrier' -require 'async/semaphore' -require 'async/http/internet' +require "async" +require "async/barrier" +require "async/semaphore" +require "async/http/internet" TOPICS = ["ruby", "python", "rust"] diff --git a/examples/google/search.rb b/examples/google/search.rb index 353199e..d976584 100755 --- a/examples/google/search.rb +++ b/examples/google/search.rb @@ -22,7 +22,7 @@ def search(term) terms = %w{thoughtful fear size payment lethal modern recognise face morning sulky mountainous contain science snow uncle skirt truthful door travel snails closed rotten halting creator teeny-tiny beautiful cherries unruly level follow strip team things suggest pretty warm end cannon bad pig consider airport strengthen youthful fog three walk furry pickle moaning fax book ruddy sigh plate cakes shame stem faulty bushes dislike train sleet one colour behavior bitter suit count loutish squeak learn watery orange idiotic seat wholesale omniscient nostalgic arithmetic instruct committee puffy program cream cake whistle rely encourage war flagrant amusing fluffy prick utter wacky occur daily son check} -if count = ENV.fetch('COUNT', 20)&.to_i +if count = ENV.fetch("COUNT", 20)&.to_i terms = terms.first(count) end @@ -40,10 +40,10 @@ def search(term) end end.map(&:wait).to_h - Console.logger.info(self, name: 'counts') {counts} + Console.logger.info(self, name: "counts") {counts} end - Console.logger.info(self, name: 'duration') {duration} + Console.logger.info(self, name: "duration") {duration} ensure google.close end diff --git a/examples/header-lowercase/benchmark.rb b/examples/header-lowercase/benchmark.rb index 4533827..27055a7 100644 --- a/examples/header-lowercase/benchmark.rb +++ b/examples/header-lowercase/benchmark.rb @@ -3,7 +3,7 @@ # Released under the MIT License. # Copyright, 2023-2024, by Samuel Williams. -require 'benchmark/ips' +require "benchmark/ips" class NormalizedHeaders def initialize(fields) @@ -26,23 +26,23 @@ def [](key) end FIELDS = { - 'content-type' => 'text/html', - 'content-length' => '127889', - 'accept-ranges' => 'bytes', - 'date' => 'Tue, 14 Jul 2015 22:00:02 GMT', - 'via' => '1.1 varnish', - 'age' => '0', - 'connection' => 'keep-alive', - 'x-served-by' => 'cache-iad2125-IAD', + "content-type" => "text/html", + "content-length" => "127889", + "accept-ranges" => "bytes", + "date" => "Tue, 14 Jul 2015 22:00:02 GMT", + "via" => "1.1 varnish", + "age" => "0", + "connection" => "keep-alive", + "x-served-by" => "cache-iad2125-IAD", } NORMALIZED_HEADERS = NormalizedHeaders.new(FIELDS) HEADERS = Headers.new(FIELDS) Benchmark.ips do |x| - x.report('NormalizedHeaders[Content-Type]') { NORMALIZED_HEADERS['Content-Type'] } - x.report('NormalizedHeaders[content-type]') { NORMALIZED_HEADERS['content-type'] } - x.report('Headers') { HEADERS['content-type'] } + x.report("NormalizedHeaders[Content-Type]") { NORMALIZED_HEADERS["Content-Type"] } + x.report("NormalizedHeaders[content-type]") { NORMALIZED_HEADERS["content-type"] } + x.report("Headers") { HEADERS["content-type"] } x.compare! end diff --git a/examples/hello/config.ru b/examples/hello/config.ru index 7179e56..fd378e7 100755 --- a/examples/hello/config.ru +++ b/examples/hello/config.ru @@ -1,10 +1,10 @@ #!/usr/bin/env falcon --verbose serve -c # frozen_string_literal: true -require 'async' -require 'async/barrier' -require 'net/http' -require 'uri' +require "async" +require "async/barrier" +require "net/http" +require "uri" run do |env| i = 1_000_000 diff --git a/examples/licenses/gemspect.rb b/examples/licenses/gemspect.rb index ca31f7a..5a02279 100755 --- a/examples/licenses/gemspect.rb +++ b/examples/licenses/gemspect.rb @@ -2,18 +2,18 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. -require 'csv' -require 'json' -require 'net/http' +require "csv" +require "json" +require "net/http" -require 'protocol/http/header/authorization' +require "protocol/http/header/authorization" class RateLimitingError < StandardError; end -@user = ENV['GITHUB_USER'] -@token = ENV['GITHUB_TOKEN'] +@user = ENV["GITHUB_USER"] +@token = ENV["GITHUB_TOKEN"] unless @user && @token fail "export GITHUB_USER and GITHUB_TOKEN!" @@ -26,8 +26,8 @@ def fetch_github_license(homepage_uri) url = URI.parse("https://api.github.com/repos/#{owner}/#{repo}/license") request = Net::HTTP::Get.new(url) - request['user-agent'] = 'fetch-github-licenses' - request['authorization'] = Protocol::HTTP::Header::Authorization.basic(@user, @token) + request["user-agent"] = "fetch-github-licenses" + request["authorization"] = Protocol::HTTP::Header::Authorization.basic(@user, @token) response = Net::HTTP.start(url.hostname) do |http| http.request(request) @@ -35,7 +35,7 @@ def fetch_github_license(homepage_uri) case response when Net::HTTPOK - JSON.parse(response.body).dig('license', 'spdx_id') + JSON.parse(response.body).dig("license", "spdx_id") when Net::HTTPNotFound, Net::HTTPMovedPermanently, Net::HTTPForbidden nil else @@ -50,7 +50,7 @@ def fetch_rubygem_license(name, version) case response when Net::HTTPOK body = JSON.parse(response.body) - [name, body.dig('licenses', 0) || fetch_github_license(body['homepage_uri'])] + [name, body.dig("licenses", 0) || fetch_github_license(body["homepage_uri"])] when Net::HTTPNotFound [name, nil] # from a non rubygems remote when Net::HTTPTooManyRequests diff --git a/examples/licenses/list.rb b/examples/licenses/list.rb index a813018..dd892e9 100755 --- a/examples/licenses/list.rb +++ b/examples/licenses/list.rb @@ -2,30 +2,30 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. -require 'csv' -require 'json' -require 'async/http/internet' +require "csv" +require "json" +require "async/http/internet" class RateLimitingError < StandardError; end @internet = Async::HTTP::Internet.new -@user = ENV['GITHUB_USER'] -@token = ENV['GITHUB_TOKEN'] +@user = ENV["GITHUB_USER"] +@token = ENV["GITHUB_TOKEN"] unless @user && @token fail "export GITHUB_USER and GITHUB_TOKEN!" end GITHUB_HEADERS = { - 'user-agent' => 'fetch-github-licenses', - 'authorization' => Protocol::HTTP::Header::Authorization.basic(@user, @token) + "user-agent" => "fetch-github-licenses", + "authorization" => Protocol::HTTP::Header::Authorization.basic(@user, @token) } RUBYGEMS_HEADERS = { - 'user-agent' => 'fetch-github-licenses' + "user-agent" => "fetch-github-licenses" } def fetch_github_license(homepage_uri) @@ -36,7 +36,7 @@ def fetch_github_license(homepage_uri) case response.status when 200 - return JSON.parse(response.read).dig('license', 'spdx_id') + return JSON.parse(response.read).dig("license", "spdx_id") when 404 return nil else @@ -52,7 +52,7 @@ def fetch_rubygem_license(name, version) case response.status when 200 body = JSON.parse(response.read) - [name, body.dig('licenses', 0) || fetch_github_license(body['homepage_uri'])] + [name, body.dig("licenses", 0) || fetch_github_license(body["homepage_uri"])] when 404 [name, nil] # from a non rubygems remote when 429 diff --git a/examples/race/client.rb b/examples/race/client.rb index 93e6c21..7c1291b 100755 --- a/examples/race/client.rb +++ b/examples/race/client.rb @@ -4,8 +4,8 @@ # Released under the MIT License. # Copyright, 2021-2024, by Samuel Williams. -require 'async' -require_relative '../../lib/async/http/internet' +require "async" +require_relative "../../lib/async/http/internet" Console.logger.fatal! @@ -16,10 +16,10 @@ 100.times do tasks << task.async { loop do - response = internet.get('http://127.0.0.1:8080/something/special') + response = internet.get("http://127.0.0.1:8080/something/special") r = response.body.join - if r.include?('nothing') - p ['something', r] + if r.include?("nothing") + p ["something", r] end end } @@ -28,10 +28,10 @@ 100.times do tasks << task.async { loop do - response = internet.get('http://127.0.0.1:8080/nothing/to/worry') + response = internet.get("http://127.0.0.1:8080/nothing/to/worry") r = response.body.join - if r.include?('something') - p ['nothing', r] + if r.include?("something") + p ["nothing", r] end end } diff --git a/examples/race/server.rb b/examples/race/server.rb index ac776bc..ba14ef6 100755 --- a/examples/race/server.rb +++ b/examples/race/server.rb @@ -2,14 +2,14 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2021-2023, by Samuel Williams. +# Copyright, 2021-2024, by Samuel Williams. -require 'async' -require 'async/http/server' -require 'async/http/endpoint' -require 'async/http/protocol/response' +require "async" +require "async/http/server" +require "async/http/endpoint" +require "async/http/protocol/response" -endpoint = Async::HTTP::Endpoint.parse('http://127.0.0.1:8080') +endpoint = Async::HTTP::Endpoint.parse("http://127.0.0.1:8080") app = lambda do |request| Protocol::HTTP::Response[200, {}, [request.path[1..-1]]] diff --git a/examples/request.rb b/examples/request.rb index 0d40f05..bd47c87 100644 --- a/examples/request.rb +++ b/examples/request.rb @@ -2,11 +2,11 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. -require 'async' -require 'async/http/client' -require 'async/http/endpoint' +require "async" +require "async/http/client" +require "async/http/endpoint" # Console.logger.level = Logger::DEBUG @@ -16,7 +16,7 @@ client = Async::HTTP::Client.new(endpoint) headers = { - 'accept' => 'text/html', + "accept" => "text/html", } request = Protocol::HTTP::Request.new(client.scheme, "www.google.com", "GET", "/search?q=cats", headers) diff --git a/examples/request/http10.rb b/examples/request/http10.rb index 7583571..2df45bc 100644 --- a/examples/request/http10.rb +++ b/examples/request/http10.rb @@ -2,11 +2,11 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. -require 'async' -require_relative '../../lib/async/http/endpoint' -require '../../lib/async/http/client' +require "async" +require_relative "../../lib/async/http/endpoint" +require "../../lib/async/http/client" Async do endpoint = Async::HTTP::Endpoint.parse("https://programming.dojo.net.nz", protocol: Async::HTTP::Protocol::HTTP10) diff --git a/examples/stream/stop.rb b/examples/stream/stop.rb index 8565fe1..c31f5a1 100644 --- a/examples/stream/stop.rb +++ b/examples/stream/stop.rb @@ -2,10 +2,10 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. -require 'async' -require 'async/http/internet' +require "async" +require "async/http/internet" Async do |parent| internet = Async::HTTP::Internet.new diff --git a/examples/trenni/Gemfile b/examples/trenni/Gemfile index d7bdde9..95bb34a 100644 --- a/examples/trenni/Gemfile +++ b/examples/trenni/Gemfile @@ -1,6 +1,6 @@ # frozen_string_literal: true -source 'https://rubygems.org' +source "https://rubygems.org" gem "trenni" gem "async-http" diff --git a/examples/trenni/streaming.rb b/examples/trenni/streaming.rb index 331dcc9..2dc3f30 100644 --- a/examples/trenni/streaming.rb +++ b/examples/trenni/streaming.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2019-2023, by Samuel Williams. +# Copyright, 2019-2024, by Samuel Williams. -require 'trenni/template' +require "trenni/template" -require 'async' -require 'async/http/body/writable' +require "async" +require "async/http/body/writable" # The template, using inline text. The sleep could be anything - database query, HTTP request, redis, etc. buffer = Trenni::Buffer.new(<<-EOF) @@ -28,7 +28,7 @@ body = Async::HTTP::Body::Writable.new generator = Async do - template.to_string({count: 100, drink: 'coffee'}, body) + template.to_string({count: 100, drink: "coffee"}, body) end while chunk = body.read diff --git a/examples/upload/client.rb b/examples/upload/client.rb index 558581d..265ae2a 100644 --- a/examples/upload/client.rb +++ b/examples/upload/client.rb @@ -2,15 +2,15 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. # Copyright, 2020, by Bruno Sutic. $LOAD_PATH.unshift File.expand_path("../../lib", __dir__) -require 'async' -require 'protocol/http/body/file' -require 'async/http/client' -require 'async/http/endpoint' +require "async" +require "protocol/http/body/file" +require "async/http/client" +require "async/http/endpoint" class Delayed < ::Protocol::HTTP::Body::Wrapper def initialize(body, delay = 0.01) @@ -35,7 +35,7 @@ def read client = Async::HTTP::Client.new(endpoint, protocol: Async::HTTP::Protocol::HTTP2) headers = [ - ['accept', 'text/plain'], + ["accept", "text/plain"], ] body = Delayed.new(Protocol::HTTP::Body::File.open(File.join(__dir__, "data.txt"), block_size: 32)) diff --git a/examples/upload/server.rb b/examples/upload/server.rb index dc7c818..b94e72c 100644 --- a/examples/upload/server.rb +++ b/examples/upload/server.rb @@ -1,19 +1,19 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. # Copyright, 2020, by Bruno Sutic. $LOAD_PATH.unshift File.expand_path("../../lib", __dir__) -require 'logger' +require "logger" -require 'async' -require 'async/http/server' -require 'async/http/endpoint' +require "async" +require "async/http/server" +require "async/http/endpoint" protocol = Async::HTTP::Protocol::HTTP2 -endpoint = Async::HTTP::Endpoint.parse('http://127.0.0.1:9222', reuse_port: true) +endpoint = Async::HTTP::Endpoint.parse("http://127.0.0.1:9222", reuse_port: true) Console.logger.level = Logger::DEBUG diff --git a/examples/upload/upload.rb b/examples/upload/upload.rb index 0ab07dc..ae9539d 100644 --- a/examples/upload/upload.rb +++ b/examples/upload/upload.rb @@ -2,18 +2,18 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. # Copyright, 2020, by Bruno Sutic. -require 'async' -require 'protocol/http/body/file' -require 'async/http/internet' +require "async" +require "protocol/http/body/file" +require "async/http/internet" Async do internet = Async::HTTP::Internet.new headers = [ - ['accept', 'text/plain'], + ["accept", "text/plain"], ] body = Protocol::HTTP::Body::File.open(File.join(__dir__, "data.txt")) diff --git a/fixtures/async/http/a_protocol.rb b/fixtures/async/http/a_protocol.rb index 5248333..82eb3de 100644 --- a/fixtures/async/http/a_protocol.rb +++ b/fixtures/async/http/a_protocol.rb @@ -4,19 +4,19 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2020, by Igor Sidorov. -require 'async' -require 'async/variable' -require 'async/clock' -require 'async/http/client' -require 'async/http/server' -require 'async/http/endpoint' -require 'async/http/body/hijack' -require 'tempfile' +require "async" +require "async/variable" +require "async/clock" +require "async/http/client" +require "async/http/server" +require "async/http/endpoint" +require "async/http/body/hijack" +require "tempfile" -require 'protocol/http/body/file' +require "protocol/http/body/file" require "protocol/http/body/buffered" -require 'sus/fixtures/async/http' +require "sus/fixtures/async/http" module Async module HTTP @@ -29,8 +29,8 @@ module HTTP expect(client.scheme).to be == "http" end - with '#close' do - it 'can close the connection' do + with "#close" do + it "can close the connection" do Async do |task| response = client.get("/") expect(response).to be(:success?) @@ -100,7 +100,7 @@ module HTTP end end - with 'buffered body' do + with "buffered body" do let(:body) {::Protocol::HTTP::Body::Buffered.new(["Hello World"])} let(:response) {::Protocol::HTTP::Response[200, {}, body]} @@ -118,30 +118,30 @@ module HTTP end end - with 'empty body' do + with "empty body" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| ::Protocol::HTTP::Response[204] end end - it 'properly handles no content responses' do + it "properly handles no content responses" do expect(client.get("/", {}).read).to be_nil end end - with 'with request trailer' do + with "with request trailer" do let(:request_received) {Async::Variable.new} let(:app) do ::Protocol::HTTP::Middleware.for do |request| - if trailer = request.headers['trailer'] - expect(request.headers).not.to have_keys('etag') + if trailer = request.headers["trailer"] + expect(request.headers).not.to have_keys("etag") request_received.value = true request.finish - expect(request.headers).to have_keys('etag') + expect(request.headers).to have_keys("etag") ::Protocol::HTTP::Response[200, [], "request trailer"] else @@ -154,14 +154,14 @@ module HTTP skip "Protocol does not support trailers!" unless subject.bidirectional? headers = ::Protocol::HTTP::Headers.new - headers.add('trailer', 'etag') + headers.add("trailer", "etag") body = Async::HTTP::Body::Writable.new Async do |task| body.write("Hello") request_received.wait - headers.add('etag', 'abcd') + headers.add("etag", "abcd") body.close_write end @@ -178,7 +178,7 @@ module HTTP let(:app) do ::Protocol::HTTP::Middleware.for do |request| headers = ::Protocol::HTTP::Headers.new - headers.add('trailer', 'etag') + headers.add("trailer", "etag") body = Async::HTTP::Body::Writable.new @@ -186,7 +186,7 @@ module HTTP body.write("response trailer") response_received.wait - headers.add('etag', 'abcd') + headers.add("etag", "abcd") body.close_write end @@ -199,9 +199,9 @@ module HTTP skip "Protocol does not support trailers!" unless subject.bidirectional? response = client.get("/") - expect(response.headers).to have_keys('trailer') + expect(response.headers).to have_keys("trailer") headers = response.headers - expect(headers).not.to have_keys('etag') + expect(headers).not.to have_keys("etag") response_received.value = true @@ -209,20 +209,20 @@ module HTTP expect(response).to be(:success?) # It was sent as a trailer. - expect(headers).to have_keys('etag') + expect(headers).to have_keys("etag") end end - with 'with working server' do + with "with working server" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| - if request.method == 'POST' + if request.method == "POST" # We stream the request body directly to the response. ::Protocol::HTTP::Response[200, {}, request.body] - elsif request.method == 'GET' + elsif request.method == "GET" expect(request.body).to be_nil - ::Protocol::HTTP::Response[200, {'my-header' => 'my-value'}, ["#{request.method} #{request.version}"]] + ::Protocol::HTTP::Response[200, {"my-header" => "my-value"}, ["#{request.method} #{request.version}"]] else ::Protocol::HTTP::Response[200, {}, ["Hello World"]] end @@ -252,7 +252,7 @@ module HTTP # reactor.print_hierarchy end - with 'using GET method' do + with "using GET method" do let(:expected) {"GET #{protocol::VERSION}"} it "can handle many simultaneous requests" do @@ -276,7 +276,7 @@ module HTTP inform "Duration: #{duration.round(2)}" end - with 'with response' do + with "with response" do let(:response) {client.get("/")} after do @@ -306,7 +306,7 @@ module HTTP end it "has response header" do - expect(response.headers['my-header']).to be == ['my-value'] + expect(response.headers["my-header"]).to be == ["my-value"] end it "has protocol version" do @@ -315,7 +315,7 @@ module HTTP end end - with 'HEAD' do + with "HEAD" do let(:response) {client.head("/")} it "is successful and without body" do @@ -327,7 +327,7 @@ module HTTP end end - with 'POST' do + with "POST" do let(:response) {client.post("/", {}, ["Hello", " ", "World"])} after do @@ -349,7 +349,7 @@ module HTTP end it "should not contain content-length response header" do - expect(response.headers).not.to have_keys('content-length') + expect(response.headers).not.to have_keys("content-length") end it "fails gracefully when closing connection" do @@ -360,7 +360,7 @@ module HTTP end end - with 'content length' do + with "content length" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| ::Protocol::HTTP::Response[200, [], ["Content Length: #{request.body.length}"]] @@ -376,7 +376,7 @@ module HTTP end end - with 'hijack with nil response' do + with "hijack with nil response" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| nil @@ -390,7 +390,7 @@ module HTTP end end - with 'partial hijack' do + with "partial hijack" do let(:content) {"Hello World!"} let(:app) do @@ -410,7 +410,7 @@ module HTTP end end - with 'body with incorrect length' do + with "body with incorrect length" do let(:bad_body) {::Protocol::HTTP::Body::Buffered.new(["Borked"], 10)} let(:app) do @@ -428,7 +428,7 @@ module HTTP end end - with 'streaming server' do + with "streaming server" do let(:sent_chunks) {[]} let(:app) do @@ -464,7 +464,7 @@ module HTTP end end - with 'hijack server' do + with "hijack server" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| if request.hijack? @@ -485,10 +485,10 @@ module HTTP end end - with 'broken server' do + with "broken server" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| - raise RuntimeError.new('simulated failure') + raise RuntimeError.new("simulated failure") end end @@ -499,7 +499,7 @@ module HTTP end end - with 'slow server' do + with "slow server" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| sleep(endpoint.timeout * 2) @@ -516,7 +516,7 @@ module HTTP end end - with 'bi-directional streaming' do + with "bi-directional streaming" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| # Echo the request body back to the client. @@ -556,7 +556,7 @@ module HTTP end end - with 'multiple client requests' do + with "multiple client requests" do let(:app) do ::Protocol::HTTP::Middleware.for do |request| ::Protocol::HTTP::Response[200, {}, [request.path]] @@ -573,13 +573,13 @@ module HTTP tasks << child loop do - response = client.get('/a') + response = client.get("/a") response.finish ensure response&.close end ensure - stopped << 'a' + stopped << "a" end end @@ -588,13 +588,13 @@ module HTTP tasks << child loop do - response = client.get('/b') + response = client.get("/b") response.finish ensure response&.close end ensure - stopped << 'b' + stopped << "b" end end diff --git a/gems.rb b/gems.rb index 4094e8a..a0b8eab 100644 --- a/gems.rb +++ b/gems.rb @@ -3,7 +3,7 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. -source 'https://rubygems.org' +source "https://rubygems.org" gemspec diff --git a/lib/async/http.rb b/lib/async/http.rb index c18c359..97f33d7 100644 --- a/lib/async/http.rb +++ b/lib/async/http.rb @@ -3,11 +3,11 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. -require_relative 'http/version' +require_relative "http/version" -require_relative 'http/client' -require_relative 'http/server' +require_relative "http/client" +require_relative "http/server" -require_relative 'http/internet' +require_relative "http/internet" -require_relative 'http/endpoint' +require_relative "http/endpoint" diff --git a/lib/async/http/body.rb b/lib/async/http/body.rb index 82b7279..1ed784b 100644 --- a/lib/async/http/body.rb +++ b/lib/async/http/body.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. -require 'protocol/http/body/buffered' -require_relative 'body/writable' +require "protocol/http/body/buffered" +require_relative "body/writable" module Async module HTTP diff --git a/lib/async/http/body/finishable.rb b/lib/async/http/body/finishable.rb index 4dcfe03..38de56c 100644 --- a/lib/async/http/body/finishable.rb +++ b/lib/async/http/body/finishable.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2019-2023, by Samuel Williams. +# Copyright, 2024, by Samuel Williams. -require 'protocol/http/body/wrapper' -require 'async/variable' +require "protocol/http/body/wrapper" +require "async/variable" module Async module HTTP diff --git a/lib/async/http/body/hijack.rb b/lib/async/http/body/hijack.rb index 769c45b..f205c48 100644 --- a/lib/async/http/body/hijack.rb +++ b/lib/async/http/body/hijack.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2019-2023, by Samuel Williams. +# Copyright, 2019-2024, by Samuel Williams. -require 'protocol/http/body/readable' -require 'protocol/http/body/stream' +require "protocol/http/body/readable" +require "protocol/http/body/stream" -require_relative 'writable' +require_relative "writable" module Async module HTTP diff --git a/lib/async/http/body/pipe.rb b/lib/async/http/body/pipe.rb index 93d7a45..6ca2d62 100644 --- a/lib/async/http/body/pipe.rb +++ b/lib/async/http/body/pipe.rb @@ -4,7 +4,7 @@ # Copyright, 2019-2024, by Samuel Williams. # Copyright, 2020, by Bruno Sutic. -require_relative 'writable' +require_relative "writable" module Async module HTTP diff --git a/lib/async/http/body/writable.rb b/lib/async/http/body/writable.rb index 18f7137..3eca79f 100644 --- a/lib/async/http/body/writable.rb +++ b/lib/async/http/body/writable.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. -require 'protocol/http/body/writable' -require 'async/queue' +require "protocol/http/body/writable" +require "async/queue" module Async module HTTP diff --git a/lib/async/http/client.rb b/lib/async/http/client.rb index 7c2c8a8..b1ab9c9 100755 --- a/lib/async/http/client.rb +++ b/lib/async/http/client.rb @@ -4,17 +4,17 @@ # Copyright, 2017-2024, by Samuel Williams. # Copyright, 2022, by Ian Ker-Seymer. -require 'io/endpoint' +require "io/endpoint" -require 'async/pool/controller' +require "async/pool/controller" -require 'protocol/http/body/completable' -require 'protocol/http/methods' +require "protocol/http/body/completable" +require "protocol/http/methods" -require 'traces/provider' +require "traces/provider" -require_relative 'protocol' -require_relative 'body/finishable' +require_relative "protocol" +require_relative "body/finishable" module Async module HTTP @@ -152,30 +152,30 @@ def call(request) } if protocol = request.protocol - attributes['http.protocol'] = protocol + attributes["http.protocol"] = protocol end if length = request.body&.length - attributes['http.request.length'] = length + attributes["http.request.length"] = length end - Traces.trace('async.http.client.call', attributes: attributes) do |span| + Traces.trace("async.http.client.call", attributes: attributes) do |span| if context = Traces.trace_context - request.headers['traceparent'] = context.to_s + request.headers["traceparent"] = context.to_s # request.headers['tracestate'] = context.state end super.tap do |response| if version = response&.version - span['http.version'] = version + span["http.version"] = version end if status = response&.status - span['http.status_code'] = status + span["http.status_code"] = status end if length = response.body&.length - span['http.response.length'] = length + span["http.response.length"] = length end end end diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index 4fba7d7..614f1b0 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -7,22 +7,22 @@ # Copyright, 2024, by Igor Sidorov. # Copyright, 2024, by Hal Brodigan. -require 'io/endpoint' -require 'io/endpoint/host_endpoint' -require 'io/endpoint/ssl_endpoint' +require "io/endpoint" +require "io/endpoint/host_endpoint" +require "io/endpoint/ssl_endpoint" -require_relative 'protocol/http' -require_relative 'protocol/https' +require_relative "protocol/http" +require_relative "protocol/https" module Async module HTTP # Represents a way to connect to a remote HTTP server. class Endpoint < ::IO::Endpoint::Generic SCHEMES = { - 'http' => URI::HTTP, - 'https' => URI::HTTPS, - 'ws' => URI::WS, - 'wss' => URI::WSS, + "http" => URI::HTTP, + "https" => URI::HTTPS, + "ws" => URI::WS, + "wss" => URI::WSS, } def self.parse(string, endpoint = nil, **options) @@ -102,7 +102,7 @@ def address end def secure? - ['https', 'wss'].include?(self.scheme) + ["https", "wss"].include?(self.scheme) end def protocol diff --git a/lib/async/http/internet.rb b/lib/async/http/internet.rb index 0a3622a..8842e18 100644 --- a/lib/async/http/internet.rb +++ b/lib/async/http/internet.rb @@ -4,12 +4,12 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2024, by Igor Sidorov. -require_relative 'client' -require_relative 'endpoint' +require_relative "client" +require_relative "endpoint" -require 'protocol/http/middleware' -require 'protocol/http/body/buffered' -require 'protocol/http/accept_encoding' +require "protocol/http/middleware" +require "protocol/http/body/buffered" +require "protocol/http/accept_encoding" module Async module HTTP diff --git a/lib/async/http/internet/instance.rb b/lib/async/http/internet/instance.rb index 1ea5832..92a60c0 100644 --- a/lib/async/http/internet/instance.rb +++ b/lib/async/http/internet/instance.rb @@ -3,7 +3,7 @@ # Released under the MIT License. # Copyright, 2021-2024, by Samuel Williams. -require_relative '../internet' +require_relative "../internet" ::Thread.attr_accessor :async_http_internet_instance diff --git a/lib/async/http/middleware/location_redirector.rb b/lib/async/http/middleware/location_redirector.rb index ae8c24f..b81bfa6 100644 --- a/lib/async/http/middleware/location_redirector.rb +++ b/lib/async/http/middleware/location_redirector.rb @@ -3,10 +3,10 @@ # Released under the MIT License. # Copyright, 2024, by Samuel Williams. -require_relative '../reference' +require_relative "../reference" -require 'protocol/http/middleware' -require 'protocol/http/body/rewindable' +require "protocol/http/middleware" +require "protocol/http/body/rewindable" module Async module HTTP @@ -34,10 +34,10 @@ class TooManyRedirects < StandardError # Header keys which should be deleted when changing a request from a POST to a GET as defined by . PROHIBITED_GET_HEADERS = [ - 'content-encoding', - 'content-language', - 'content-location', - 'content-type', + "content-encoding", + "content-language", + "content-location", + "content-type", ] # maximum_hops is the max number of redirects. Set to 0 to allow 1 request with no redirects. @@ -91,7 +91,7 @@ def call(request) hops += 1 # Get the redirect location: - unless location = response.headers['location'] + unless location = response.headers["location"] return response end diff --git a/lib/async/http/mock.rb b/lib/async/http/mock.rb index d9ecae3..ec01aae 100644 --- a/lib/async/http/mock.rb +++ b/lib/async/http/mock.rb @@ -3,4 +3,4 @@ # Released under the MIT License. # Copyright, 2024, by Samuel Williams. -require_relative 'mock/endpoint' +require_relative "mock/endpoint" diff --git a/lib/async/http/mock/endpoint.rb b/lib/async/http/mock/endpoint.rb index 91c9279..ff3ab81 100644 --- a/lib/async/http/mock/endpoint.rb +++ b/lib/async/http/mock/endpoint.rb @@ -3,9 +3,9 @@ # Released under the MIT License. # Copyright, 2024, by Samuel Williams. -require_relative '../protocol' +require_relative "../protocol" -require 'async/queue' +require "async/queue" module Async module HTTP diff --git a/lib/async/http/protocol.rb b/lib/async/http/protocol.rb index b61afe1..6eb5e27 100644 --- a/lib/async/http/protocol.rb +++ b/lib/async/http/protocol.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2023, by Samuel Williams. +# Copyright, 2017-2024, by Samuel Williams. -require_relative 'protocol/http1' -require_relative 'protocol/https' +require_relative "protocol/http1" +require_relative "protocol/https" module Async module HTTP diff --git a/lib/async/http/protocol/http.rb b/lib/async/http/protocol/http.rb index 6b57c08..b70bb83 100644 --- a/lib/async/http/protocol/http.rb +++ b/lib/async/http/protocol/http.rb @@ -4,8 +4,8 @@ # Copyright, 2024, by Thomas Morgan. # Copyright, 2024, by Samuel Williams. -require_relative 'http1' -require_relative 'http2' +require_relative "http1" +require_relative "http2" module Async module HTTP diff --git a/lib/async/http/protocol/http1.rb b/lib/async/http/protocol/http1.rb index e51b57b..2d7b23a 100644 --- a/lib/async/http/protocol/http1.rb +++ b/lib/async/http/protocol/http1.rb @@ -4,10 +4,10 @@ # Copyright, 2017-2024, by Samuel Williams. # Copyright, 2024, by Thomas Morgan. -require_relative 'http1/client' -require_relative 'http1/server' +require_relative "http1/client" +require_relative "http1/server" -require 'io/stream' +require "io/stream" module Async module HTTP diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index c34eef0..ec295f2 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -3,7 +3,7 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require_relative 'connection' +require_relative "connection" module Async module HTTP diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index 0f88c15..bfaf0d7 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -3,10 +3,10 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require 'protocol/http1' +require "protocol/http1" -require_relative 'request' -require_relative 'response' +require_relative "request" +require_relative "response" module Async module HTTP diff --git a/lib/async/http/protocol/http1/request.rb b/lib/async/http/protocol/http1/request.rb index d70ee3e..9408be2 100644 --- a/lib/async/http/protocol/http1/request.rb +++ b/lib/async/http/protocol/http1/request.rb @@ -3,7 +3,7 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require_relative '../request' +require_relative "../request" module Async module HTTP @@ -16,13 +16,13 @@ def self.read(connection) end end - UPGRADE = 'upgrade' + UPGRADE = "upgrade" def initialize(connection, authority, method, path, version, headers, body) @connection = connection # HTTP/1 requests with an upgrade header (which can contain zero or more values) are extracted into the protocol field of the request, and we expect a response to select one of those protocols with a status code of 101 Switching Protocols. - protocol = headers.delete('upgrade') + protocol = headers.delete("upgrade") super(nil, authority, method, path, version, headers, body, protocol, self.public_method(:write_interim_response)) end diff --git a/lib/async/http/protocol/http1/response.rb b/lib/async/http/protocol/http1/response.rb index 08afc35..0fbd8fc 100644 --- a/lib/async/http/protocol/http1/response.rb +++ b/lib/async/http/protocol/http1/response.rb @@ -4,7 +4,7 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2023, by Josh Huber. -require_relative '../response' +require_relative "../response" module Async module HTTP @@ -23,7 +23,7 @@ def self.read(connection, request) end end - UPGRADE = 'upgrade' + UPGRADE = "upgrade" # @attribute [String] The HTTP response line reason. attr :reason diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index b30e0c6..fd333c8 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -6,10 +6,10 @@ # Copyright, 2023, by Thomas Morgan. # Copyright, 2024, by Anton Zhuravsky. -require_relative 'connection' -require_relative '../../body/finishable' +require_relative "connection" +require_relative "../../body/finishable" -require 'console/event/failure' +require "console/event/failure" module Async module HTTP diff --git a/lib/async/http/protocol/http10.rb b/lib/async/http/protocol/http10.rb index 793048a..e37308b 100755 --- a/lib/async/http/protocol/http10.rb +++ b/lib/async/http/protocol/http10.rb @@ -4,7 +4,7 @@ # Copyright, 2017-2024, by Samuel Williams. # Copyright, 2024, by Thomas Morgan. -require_relative 'http1' +require_relative "http1" module Async module HTTP diff --git a/lib/async/http/protocol/http11.rb b/lib/async/http/protocol/http11.rb index dd8135f..b29f246 100644 --- a/lib/async/http/protocol/http11.rb +++ b/lib/async/http/protocol/http11.rb @@ -5,7 +5,7 @@ # Copyright, 2018, by Janko Marohnić. # Copyright, 2024, by Thomas Morgan. -require_relative 'http1' +require_relative "http1" module Async module HTTP diff --git a/lib/async/http/protocol/http2.rb b/lib/async/http/protocol/http2.rb index 96da7e2..3222f00 100644 --- a/lib/async/http/protocol/http2.rb +++ b/lib/async/http/protocol/http2.rb @@ -4,10 +4,10 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2024, by Thomas Morgan. -require_relative 'http2/client' -require_relative 'http2/server' +require_relative "http2/client" +require_relative "http2/server" -require 'io/stream' +require "io/stream" module Async module HTTP diff --git a/lib/async/http/protocol/http2/client.rb b/lib/async/http/protocol/http2/client.rb index e55cf9c..ba6e979 100644 --- a/lib/async/http/protocol/http2/client.rb +++ b/lib/async/http/protocol/http2/client.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. -require_relative 'connection' -require_relative 'response' +require_relative "connection" +require_relative "response" -require 'protocol/http2/client' +require "protocol/http2/client" module Async module HTTP diff --git a/lib/async/http/protocol/http2/connection.rb b/lib/async/http/protocol/http2/connection.rb index cbab8f7..7d989d0 100644 --- a/lib/async/http/protocol/http2/connection.rb +++ b/lib/async/http/protocol/http2/connection.rb @@ -4,25 +4,25 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2020, by Bruno Sutic. -require_relative 'stream' +require_relative "stream" -require 'async/semaphore' +require "async/semaphore" module Async module HTTP module Protocol module HTTP2 - HTTPS = 'https'.freeze - SCHEME = ':scheme'.freeze - METHOD = ':method'.freeze - PATH = ':path'.freeze - AUTHORITY = ':authority'.freeze - STATUS = ':status'.freeze - PROTOCOL = ':protocol'.freeze + HTTPS = "https".freeze + SCHEME = ":scheme".freeze + METHOD = ":method".freeze + PATH = ":path".freeze + AUTHORITY = ":authority".freeze + STATUS = ":status".freeze + PROTOCOL = ":protocol".freeze - CONTENT_LENGTH = 'content-length'.freeze - CONNECTION = 'connection'.freeze - TRAILER = 'trailer'.freeze + CONTENT_LENGTH = "content-length".freeze + CONNECTION = "connection".freeze + TRAILER = "trailer".freeze module Connection def initialize(*) diff --git a/lib/async/http/protocol/http2/input.rb b/lib/async/http/protocol/http2/input.rb index 116681b..7365b0d 100644 --- a/lib/async/http/protocol/http2/input.rb +++ b/lib/async/http/protocol/http2/input.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. -require 'protocol/http/body/writable' +require "protocol/http/body/writable" module Async module HTTP diff --git a/lib/async/http/protocol/http2/output.rb b/lib/async/http/protocol/http2/output.rb index 4e3d5cf..2f2efd4 100644 --- a/lib/async/http/protocol/http2/output.rb +++ b/lib/async/http/protocol/http2/output.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. -require 'protocol/http/body/stream' +require "protocol/http/body/stream" module Async module HTTP diff --git a/lib/async/http/protocol/http2/request.rb b/lib/async/http/protocol/http2/request.rb index 406ca2e..84fdd5a 100644 --- a/lib/async/http/protocol/http2/request.rb +++ b/lib/async/http/protocol/http2/request.rb @@ -3,8 +3,8 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require_relative '../request' -require_relative 'stream' +require_relative "../request" +require_relative "stream" module Async module HTTP @@ -53,7 +53,7 @@ def receive_initial_headers(headers, end_stream) @length = Integer(value) elsif key == CONNECTION raise ::Protocol::HTTP2::HeaderError, "Connection header is not allowed!" - elsif key.start_with? ':' + elsif key.start_with? ":" raise ::Protocol::HTTP2::HeaderError, "Invalid pseudo-header #{key}!" elsif key =~ /[A-Z]/ raise ::Protocol::HTTP2::HeaderError, "Invalid characters in header #{key}!" @@ -107,7 +107,7 @@ def hijack? end NO_RESPONSE = [ - [STATUS, '500'], + [STATUS, "500"], ] def send_response(response) diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index 923b5c0..c952c47 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -3,8 +3,8 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require_relative '../response' -require_relative 'stream' +require_relative "../response" +require_relative "stream" module Async module HTTP @@ -41,7 +41,7 @@ def receive_initial_headers(headers, end_stream) # While in theory, the response pseudo-headers may be extended in the future, currently they only response pseudo-header is :status, so we can assume it is always the first header. status_header = headers.shift - if status_header.first != ':status' + if status_header.first != ":status" raise ProtocolError, "Invalid response headers: #{headers.inspect}" end @@ -185,7 +185,7 @@ def build_request(headers) raise ::Protocol::HTTP2::HeaderError, "Request path already specified!" if request.path request.path = value - elsif key.start_with? ':' + elsif key.start_with? ":" raise ::Protocol::HTTP2::HeaderError, "Invalid pseudo-header #{key}!" else request.headers[key] = value diff --git a/lib/async/http/protocol/http2/server.rb b/lib/async/http/protocol/http2/server.rb index a0b77ad..0372955 100644 --- a/lib/async/http/protocol/http2/server.rb +++ b/lib/async/http/protocol/http2/server.rb @@ -3,10 +3,10 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require_relative 'connection' -require_relative 'request' +require_relative "connection" +require_relative "request" -require 'protocol/http2/server' +require "protocol/http2/server" module Async module HTTP diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index c9a98ee..505196d 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -5,10 +5,10 @@ # Copyright, 2022, by Marco Concetto Rudilosso. # Copyright, 2023, by Thomas Morgan. -require 'protocol/http2/stream' +require "protocol/http2/stream" -require_relative 'input' -require_relative 'output' +require_relative "input" +require_relative "output" module Async module HTTP @@ -39,7 +39,7 @@ def initialize(*) def add_header(key, value) if key == CONNECTION raise ::Protocol::HTTP2::HeaderError, "Connection header is not allowed!" - elsif key.start_with? ':' + elsif key.start_with? ":" raise ::Protocol::HTTP2::HeaderError, "Invalid pseudo-header #{key}!" elsif key =~ /[A-Z]/ raise ::Protocol::HTTP2::HeaderError, "Invalid upper-case characters in header #{key}!" diff --git a/lib/async/http/protocol/https.rb b/lib/async/http/protocol/https.rb index 028885e..fd5b07e 100644 --- a/lib/async/http/protocol/https.rb +++ b/lib/async/http/protocol/https.rb @@ -4,10 +4,10 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2019, by Brian Morearty. -require_relative 'http10' -require_relative 'http11' +require_relative "http10" +require_relative "http11" -require_relative 'http2' +require_relative "http2" module Async module HTTP diff --git a/lib/async/http/protocol/request.rb b/lib/async/http/protocol/request.rb index ded0d2c..fabfb42 100644 --- a/lib/async/http/protocol/request.rb +++ b/lib/async/http/protocol/request.rb @@ -3,10 +3,10 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. -require 'protocol/http/request' -require 'protocol/http/headers' +require "protocol/http/request" +require "protocol/http/headers" -require_relative '../body/writable' +require_relative "../body/writable" module Async module HTTP diff --git a/lib/async/http/protocol/response.rb b/lib/async/http/protocol/response.rb index a8f765d..e55998c 100644 --- a/lib/async/http/protocol/response.rb +++ b/lib/async/http/protocol/response.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2023, by Samuel Williams. +# Copyright, 2017-2024, by Samuel Williams. -require 'protocol/http/response' +require "protocol/http/response" -require_relative '../body/writable' +require_relative "../body/writable" module Async module HTTP diff --git a/lib/async/http/proxy.rb b/lib/async/http/proxy.rb index d1d7bee..b6fd204 100644 --- a/lib/async/http/proxy.rb +++ b/lib/async/http/proxy.rb @@ -3,10 +3,10 @@ # Released under the MIT License. # Copyright, 2019-2024, by Samuel Williams. -require_relative 'client' -require_relative 'endpoint' +require_relative "client" +require_relative "endpoint" -require_relative 'body/pipe' +require_relative "body/pipe" module Async module HTTP diff --git a/lib/async/http/reference.rb b/lib/async/http/reference.rb index ba9a36e..0e9775d 100644 --- a/lib/async/http/reference.rb +++ b/lib/async/http/reference.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. -require 'protocol/http/reference' +require "protocol/http/reference" module Async module HTTP diff --git a/lib/async/http/relative_location.rb b/lib/async/http/relative_location.rb index e99d383..42ce9e8 100644 --- a/lib/async/http/relative_location.rb +++ b/lib/async/http/relative_location.rb @@ -4,7 +4,7 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2019-2020, by Brian Morearty. -require_relative 'middleware/location_redirector' +require_relative "middleware/location_redirector" warn "`Async::HTTP::RelativeLocation` is deprecated and will be removed in the next release. Please use `Async::HTTP::Middleware::LocationRedirector` instead.", uplevel: 1 diff --git a/lib/async/http/server.rb b/lib/async/http/server.rb index 702d393..bb7c515 100755 --- a/lib/async/http/server.rb +++ b/lib/async/http/server.rb @@ -4,12 +4,12 @@ # Copyright, 2017-2024, by Samuel Williams. # Copyright, 2019, by Brian Morearty. -require 'async' -require 'io/endpoint' -require 'protocol/http/middleware' -require 'traces/provider' +require "async" +require "io/endpoint" +require "protocol/http/middleware" +require "traces/provider" -require_relative 'protocol' +require_relative "protocol" module Async module HTTP @@ -76,8 +76,8 @@ def run Traces::Provider(self) do def call(request) - if trace_parent = request.headers['traceparent'] - Traces.trace_context = Traces::Context.parse(trace_parent.join, request.headers['tracestate'], remote: true) + if trace_parent = request.headers["traceparent"] + Traces.trace_context = Traces::Context.parse(trace_parent.join, request.headers["tracestate"], remote: true) end attributes = { @@ -86,25 +86,25 @@ def call(request) 'http.authority': request.authority, 'http.scheme': request.scheme, 'http.path': request.path, - 'http.user_agent': request.headers['user-agent'], + 'http.user_agent': request.headers["user-agent"], } if length = request.body&.length - attributes['http.request.length'] = length + attributes["http.request.length"] = length end if protocol = request.protocol - attributes['http.protocol'] = protocol + attributes["http.protocol"] = protocol end - Traces.trace('async.http.server.call', resource: "#{request.method} #{request.path}", attributes: attributes) do |span| + Traces.trace("async.http.server.call", resource: "#{request.method} #{request.path}", attributes: attributes) do |span| super.tap do |response| if status = response&.status - span['http.status_code'] = status + span["http.status_code"] = status end if length = response&.body&.length - span['http.response.length'] = length + span["http.response.length"] = length end end end diff --git a/lib/async/http/statistics.rb b/lib/async/http/statistics.rb index a4eb6db..302847d 100644 --- a/lib/async/http/statistics.rb +++ b/lib/async/http/statistics.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. -require 'protocol/http/body/wrapper' +require "protocol/http/body/wrapper" -require 'async/clock' +require "async/clock" module Async module HTTP @@ -89,7 +89,7 @@ def to_s parts << "took #{format_duration(duration)} until first chunk" end - return parts.join('; ') + return parts.join("; ") end def inspect diff --git a/test/async/http/body.rb b/test/async/http/body.rb index 1235a54..351d486 100644 --- a/test/async/http/body.rb +++ b/test/async/http/body.rb @@ -3,16 +3,16 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require 'async/http/body' +require "async/http/body" -require 'sus/fixtures/async' -require 'sus/fixtures/openssl' -require 'sus/fixtures/async/http' -require 'localhost/authority' -require 'io/endpoint/ssl_endpoint' +require "sus/fixtures/async" +require "sus/fixtures/openssl" +require "sus/fixtures/async/http" +require "localhost/authority" +require "io/endpoint/ssl_endpoint" ABody = Sus::Shared("a body") do - with 'echo server' do + with "echo server" do let(:app) do Protocol::HTTP::Middleware.for do |request| input = request.body diff --git a/test/async/http/body/hijack.rb b/test/async/http/body/hijack.rb index 9d994a2..a15101f 100644 --- a/test/async/http/body/hijack.rb +++ b/test/async/http/body/hijack.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2019-2023, by Samuel Williams. +# Copyright, 2019-2024, by Samuel Williams. -require 'async/http/body/hijack' +require "async/http/body/hijack" -require 'sus/fixtures/async' +require "sus/fixtures/async" describe Async::HTTP::Body::Hijack do include Sus::Fixtures::Async::ReactorContext @@ -21,7 +21,7 @@ let(:content) {"Hello World!"} - with '#call' do + with "#call" do let(:stream) {Async::HTTP::Body::Writable.new} it "should generate body using direct invocation" do diff --git a/test/async/http/body/pipe.rb b/test/async/http/body/pipe.rb index 8e05263..12ddc66 100644 --- a/test/async/http/body/pipe.rb +++ b/test/async/http/body/pipe.rb @@ -4,19 +4,19 @@ # Copyright, 2020, by Bruno Sutic. # Copyright, 2020-2024, by Samuel Williams. -require 'async' -require 'async/http/body/pipe' -require 'async/http/body/writable' +require "async" +require "async/http/body/pipe" +require "async/http/body/writable" -require 'sus/fixtures/async' +require "sus/fixtures/async" describe Async::HTTP::Body::Pipe do let(:input) {Async::HTTP::Body::Writable.new} let(:pipe) {subject.new(input)} - let(:data) {'Hello World!'} + let(:data) {"Hello World!"} - with '#to_io' do + with "#to_io" do include Sus::Fixtures::Async::ReactorContext let(:input_write_duration) {0} @@ -27,7 +27,7 @@ def before # input writer task Async do |task| - first, second = data.split(' ') + first, second = data.split(" ") input.write("#{first} ") sleep(input_write_duration) if input_write_duration > 0 input.write(second) @@ -44,23 +44,23 @@ def before expect(io.read).to be == data end - with 'blocking reads' do + with "blocking reads" do let(:input_write_duration) {0.01} - it 'returns an io socket' do + it "returns an io socket" do expect(io.read).to be == data end end end - with 'reactor going out of scope' do - it 'finishes' do + with "reactor going out of scope" do + it "finishes" do # ensures pipe background tasks are transient Async{pipe} end - with 'closed pipe' do - it 'finishes' do + with "closed pipe" do + it "finishes" do Async{pipe.close} end end diff --git a/test/async/http/client.rb b/test/async/http/client.rb index 621364f..5091ed2 100644 --- a/test/async/http/client.rb +++ b/test/async/http/client.rb @@ -3,18 +3,18 @@ # Released under the MIT License. # Copyright, 2017-2024, by Samuel Williams. -require 'async/http/server' -require 'async/http/client' -require 'async/reactor' +require "async/http/server" +require "async/http/client" +require "async/reactor" -require 'async/http/endpoint' -require 'protocol/http/accept_encoding' +require "async/http/endpoint" +require "protocol/http/accept_encoding" -require 'sus/fixtures/async' -require 'sus/fixtures/async/http' +require "sus/fixtures/async" +require "sus/fixtures/async/http" describe Async::HTTP::Client do - with 'basic server' do + with "basic server" do include Sus::Fixtures::Async::HTTP::ServerContext it "client can get resource" do @@ -23,7 +23,7 @@ expect(response).to be(:success?) end - with 'client' do + with "client" do with "#as_json" do it "generates a JSON representation" do expect(client.as_json).to be == { @@ -35,13 +35,13 @@ } end - it 'generates a JSON string' do + it "generates a JSON string" do expect(JSON.dump(client)).to be == client.to_json end end end - with 'server' do + with "server" do with "#as_json" do it "generates a JSON representation" do expect(server.as_json).to be == { @@ -51,17 +51,17 @@ } end - it 'generates a JSON string' do + it "generates a JSON string" do expect(JSON.dump(server)).to be == server.to_json end end end end - with 'non-existant host' do + with "non-existant host" do include Sus::Fixtures::Async::ReactorContext - let(:endpoint) {Async::HTTP::Endpoint.parse('http://the.future')} + let(:endpoint) {Async::HTTP::Endpoint.parse("http://the.future")} let(:client) {Async::HTTP::Client.new(endpoint)} it "should fail to connect" do diff --git a/test/async/http/client/google.rb b/test/async/http/client/google.rb index cc9c1d6..5d5e369 100644 --- a/test/async/http/client/google.rb +++ b/test/async/http/client/google.rb @@ -3,17 +3,17 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require 'async/http/client' -require 'async/http/endpoint' +require "async/http/client" +require "async/http/endpoint" -require 'protocol/http/accept_encoding' +require "protocol/http/accept_encoding" -require 'sus/fixtures/async' +require "sus/fixtures/async" describe Async::HTTP::Client do include Sus::Fixtures::Async::ReactorContext - let(:endpoint) {Async::HTTP::Endpoint.parse('https://www.google.com')} + let(:endpoint) {Async::HTTP::Endpoint.parse("https://www.google.com")} let(:client) {Async::HTTP::Client.new(endpoint)} it "should specify a hostname" do @@ -21,8 +21,8 @@ expect(client.authority).to be == "www.google.com" end - it 'can fetch remote resource' do - response = client.get('/', {'accept' => '*/*'}) + it "can fetch remote resource" do + response = client.get("/", {"accept" => "*/*"}) response.finish @@ -34,11 +34,11 @@ it "can request remote resource with compression" do compressor = Protocol::HTTP::AcceptEncoding.new(client) - response = compressor.get("/", {'accept-encoding' => 'gzip'}) + response = compressor.get("/", {"accept-encoding" => "gzip"}) expect(response).to be(:success?) expect(response.body).to be_a Async::HTTP::Body::Inflate - expect(response.read).to be(:start_with?, '') + expect(response.read).to be(:start_with?, "") end end diff --git a/test/async/http/endpoint.rb b/test/async/http/endpoint.rb index e98e7c7..04a1e0e 100644 --- a/test/async/http/endpoint.rb +++ b/test/async/http/endpoint.rb @@ -5,7 +5,7 @@ # Copyright, 2021-2022, by Adam Daniels. # Copyright, 2024, by Thomas Morgan. -require 'async/http/endpoint' +require "async/http/endpoint" describe Async::HTTP::Endpoint do it "should fail to parse relative url" do @@ -14,7 +14,7 @@ end.to raise_exception(ArgumentError, message: be =~ /absolute/) end - with '#port' do + with "#port" do let(:url_string) {"https://localhost:9292"} it "extracts port from URL" do @@ -30,34 +30,34 @@ end end - with '#hostname' do + with "#hostname" do describe Async::HTTP::Endpoint.parse("https://127.0.0.1:9292") do - it 'has correct hostname' do - expect(subject).to have_attributes(hostname: be == '127.0.0.1') + it "has correct hostname" do + expect(subject).to have_attributes(hostname: be == "127.0.0.1") end it "should be connecting to 127.0.0.1" do expect(subject.endpoint).to be_a ::IO::Endpoint::SSLEndpoint - expect(subject.endpoint).to have_attributes(hostname: be == '127.0.0.1') - expect(subject.endpoint.endpoint).to have_attributes(hostname: be == '127.0.0.1') + expect(subject.endpoint).to have_attributes(hostname: be == "127.0.0.1") + expect(subject.endpoint.endpoint).to have_attributes(hostname: be == "127.0.0.1") end end - describe Async::HTTP::Endpoint.parse("https://127.0.0.1:9292", hostname: 'localhost') do - it 'has correct hostname' do - expect(subject).to have_attributes(hostname: be == 'localhost') + describe Async::HTTP::Endpoint.parse("https://127.0.0.1:9292", hostname: "localhost") do + it "has correct hostname" do + expect(subject).to have_attributes(hostname: be == "localhost") expect(subject).not.to be(:localhost?) end it "should be connecting to localhost" do expect(subject.endpoint).to be_a ::IO::Endpoint::SSLEndpoint - expect(subject.endpoint).to have_attributes(hostname: be == '127.0.0.1') - expect(subject.endpoint.endpoint).to have_attributes(hostname: be == 'localhost') + expect(subject.endpoint).to have_attributes(hostname: be == "127.0.0.1") + expect(subject.endpoint.endpoint).to have_attributes(hostname: be == "localhost") end end end - with '.for' do + with ".for" do describe Async::HTTP::Endpoint.for("http", "localhost") do it "should have correct attributes" do expect(subject).to have_attributes( @@ -82,7 +82,7 @@ end end - with 'invalid scheme' do + with "invalid scheme" do it "should raise an argument error" do expect do Async::HTTP::Endpoint.for("foo", "localhost") @@ -95,7 +95,7 @@ end end - with '#secure?' do + with "#secure?" do describe Async::HTTP::Endpoint.parse("http://localhost") do it "should not be secure" do expect(subject).not.to be(:secure?) @@ -108,8 +108,8 @@ end end - with 'scheme: https' do - describe Async::HTTP::Endpoint.parse("http://localhost", scheme: 'https') do + with "scheme: https" do + describe Async::HTTP::Endpoint.parse("http://localhost", scheme: "https") do it "should be secure" do expect(subject).to be(:secure?) end @@ -117,7 +117,7 @@ end end - with '#localhost?' do + with "#localhost?" do describe Async::HTTP::Endpoint.parse("http://localhost") do it "should be localhost" do expect(subject).to be(:localhost?) @@ -149,14 +149,14 @@ end end - with '#path' do + with "#path" do describe Async::HTTP::Endpoint.parse("http://foo.com/bar?baz") do it "should have correct path" do expect(subject).to have_attributes(path: be == "/bar?baz") end end - with 'websocket scheme' do + with "websocket scheme" do describe Async::HTTP::Endpoint.parse("wss://foo.com/bar?baz") do it "should have correct path" do expect(subject).to have_attributes(path: be == "/bar?baz") @@ -180,7 +180,7 @@ end it "should not be equal if path is different" do - other = Async::HTTP::Endpoint.parse('http://www.google.com/search?q=ruby') + other = Async::HTTP::Endpoint.parse("http://www.google.com/search?q=ruby") expect(subject).not.to be == other expect(subject).not.to be(:eql?, other) end diff --git a/test/async/http/internet.rb b/test/async/http/internet.rb index d8a9550..2b2e028 100644 --- a/test/async/http/internet.rb +++ b/test/async/http/internet.rb @@ -5,17 +5,17 @@ # Copyright, 2024, by Igor Sidorov. # Copyright, 2024, by Hal Brodigan. -require 'async/http/internet' -require 'async/reactor' +require "async/http/internet" +require "async/reactor" -require 'json' -require 'sus/fixtures/async' +require "json" +require "sus/fixtures/async" describe Async::HTTP::Internet do include Sus::Fixtures::Async::ReactorContext let(:internet) {subject.new} - let(:headers) {[['accept', '*/*'], ['user-agent', 'async-http']]} + let(:headers) {[["accept", "*/*"], ["user-agent", "async-http"]]} it "can fetch remote website" do response = internet.get("https://www.google.com/", headers) @@ -45,12 +45,12 @@ expect{JSON.parse(response.read)}.not.to raise_exception end - it 'can fetch remote website when given custom endpoint instead of url' do + it "can fetch remote website when given custom endpoint instead of url" do ssl_context = OpenSSL::SSL::SSLContext.new ssl_context.set_params(verify_mode: OpenSSL::SSL::VERIFY_NONE) # example of site with invalid certificate that will fail to be fetched without custom SSL options - endpoint = Async::HTTP::Endpoint.parse('https://expired.badssl.com', ssl_context: ssl_context) + endpoint = Async::HTTP::Endpoint.parse("https://expired.badssl.com", ssl_context: ssl_context) response = internet.get(endpoint, headers) diff --git a/test/async/http/internet/instance.rb b/test/async/http/internet/instance.rb index d254bd5..847bf74 100644 --- a/test/async/http/internet/instance.rb +++ b/test/async/http/internet/instance.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2021-2023, by Samuel Williams. +# Copyright, 2021-2024, by Samuel Williams. -require 'async/http/internet/instance' +require "async/http/internet/instance" describe Async::HTTP::Internet do - describe '.instance' do + describe ".instance" do it "returns an internet instance" do expect(Async::HTTP::Internet.instance).to be_a(Async::HTTP::Internet) end diff --git a/test/async/http/middleware/location_redirector.rb b/test/async/http/middleware/location_redirector.rb index 1efb8ee..82add40 100644 --- a/test/async/http/middleware/location_redirector.rb +++ b/test/async/http/middleware/location_redirector.rb @@ -4,51 +4,51 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2019-2020, by Brian Morearty. -require 'async/http/middleware/location_redirector' -require 'async/http/server' +require "async/http/middleware/location_redirector" +require "async/http/server" -require 'sus/fixtures/async/http' +require "sus/fixtures/async/http" describe Async::HTTP::Middleware::LocationRedirector do include Sus::Fixtures::Async::HTTP::ServerContext let(:relative_location) {subject.new(@client, 1)} - with 'server redirections' do - with '301' do + with "server redirections" do + with "301" do let(:app) do Protocol::HTTP::Middleware.for do |request| case request.path - when '/home' - Protocol::HTTP::Response[301, {'location' => '/'}, []] - when '/' - Protocol::HTTP::Response[301, {'location' => '/index.html'}, []] - when '/index.html' + when "/home" + Protocol::HTTP::Response[301, {"location" => "/"}, []] + when "/" + Protocol::HTTP::Response[301, {"location" => "/index.html"}, []] + when "/index.html" Protocol::HTTP::Response[200, {}, [request.method]] end end end - it 'should redirect POST to GET' do + it "should redirect POST to GET" do body = Protocol::HTTP::Body::Buffered.wrap(["Hello, World!"]) expect(body).to receive(:finish) - response = relative_location.post('/', {}, body) + response = relative_location.post("/", {}, body) expect(response).to be(:success?) expect(response.read).to be == "GET" end - with 'limiting redirects' do - it 'should allow the maximum number of redirects' do - response = relative_location.get('/') + with "limiting redirects" do + it "should allow the maximum number of redirects" do + response = relative_location.get("/") response.finish expect(response).to be(:success?) end - it 'should fail with maximum redirects' do + it "should fail with maximum redirects" do expect do - response = relative_location.get('/home') + response = relative_location.get("/home") end.to raise_exception(subject::TooManyRedirects, message: be =~ /maximum/) end end @@ -59,7 +59,7 @@ end it "should not follow the redirect" do - response = relative_location.get('/') + response = relative_location.get("/") response.finish expect(response).to be(:redirection?) @@ -67,60 +67,60 @@ end end - with '302' do + with "302" do let(:app) do Protocol::HTTP::Middleware.for do |request| case request.path - when '/' - Protocol::HTTP::Response[302, {'location' => '/index.html'}, []] - when '/index.html' + when "/" + Protocol::HTTP::Response[302, {"location" => "/index.html"}, []] + when "/index.html" Protocol::HTTP::Response[200, {}, [request.method]] end end end - it 'should redirect POST to GET' do - response = relative_location.post('/') + it "should redirect POST to GET" do + response = relative_location.post("/") expect(response).to be(:success?) expect(response.read).to be == "GET" end end - with '307' do + with "307" do let(:app) do Protocol::HTTP::Middleware.for do |request| case request.path - when '/' - Protocol::HTTP::Response[307, {'location' => '/index.html'}, []] - when '/index.html' + when "/" + Protocol::HTTP::Response[307, {"location" => "/index.html"}, []] + when "/index.html" Protocol::HTTP::Response[200, {}, [request.method]] end end end - it 'should redirect with same method' do - response = relative_location.post('/') + it "should redirect with same method" do + response = relative_location.post("/") expect(response).to be(:success?) expect(response.read).to be == "POST" end end - with '308' do + with "308" do let(:app) do Protocol::HTTP::Middleware.for do |request| case request.path - when '/' - Protocol::HTTP::Response[308, {'location' => '/index.html'}, []] - when '/index.html' + when "/" + Protocol::HTTP::Response[308, {"location" => "/index.html"}, []] + when "/index.html" Protocol::HTTP::Response[200, {}, [request.method]] end end end - it 'should redirect with same method' do - response = relative_location.post('/') + it "should redirect with same method" do + response = relative_location.post("/") expect(response).to be(:success?) expect(response.read).to be == "POST" diff --git a/test/async/http/mock.rb b/test/async/http/mock.rb index 8396fe1..d4da575 100644 --- a/test/async/http/mock.rb +++ b/test/async/http/mock.rb @@ -3,11 +3,11 @@ # Released under the MIT License. # Copyright, 2024, by Samuel Williams. -require 'async/http/mock' -require 'async/http/endpoint' -require 'async/http/client' +require "async/http/mock" +require "async/http/endpoint" +require "async/http/client" -require 'sus/fixtures/async/reactor_context' +require "sus/fixtures/async/reactor_context" describe Async::HTTP::Mock do include Sus::Fixtures::Async::ReactorContext @@ -29,7 +29,7 @@ expect(response.read).to be == "Hello World" end - with 'mocked client' do + with "mocked client" do it "can mock a client" do server = Async do endpoint.run do |request| diff --git a/test/async/http/protocol/http.rb b/test/async/http/protocol/http.rb index 90a2b5a..eaaa947 100755 --- a/test/async/http/protocol/http.rb +++ b/test/async/http/protocol/http.rb @@ -4,33 +4,33 @@ # Copyright, 2024, by Thomas Morgan. # Copyright, 2024, by Samuel Williams. -require 'async/http/protocol/http' -require 'async/http/a_protocol' +require "async/http/protocol/http" +require "async/http/a_protocol" describe Async::HTTP::Protocol::HTTP do - with 'server' do + with "server" do include Sus::Fixtures::Async::HTTP::ServerContext let(:protocol) {subject} - with 'http11 client' do - it 'should make a successful request' do - response = client.get('/') + with "http11 client" do + it "should make a successful request" do + response = client.get("/") expect(response).to be(:success?) - expect(response.version).to be == 'HTTP/1.1' + expect(response.version).to be == "HTTP/1.1" response.read end end - with 'http2 client' do + with "http2 client" do def make_client(endpoint, **options) options[:protocol] = Async::HTTP::Protocol::HTTP2 super end - it 'should make a successful request' do - response = client.get('/') + it "should make a successful request" do + response = client.get("/") expect(response).to be(:success?) - expect(response.version).to be == 'HTTP/2' + expect(response.version).to be == "HTTP/2" response.read end end diff --git a/test/async/http/protocol/http10.rb b/test/async/http/protocol/http10.rb index 26ae0be..6770e69 100644 --- a/test/async/http/protocol/http10.rb +++ b/test/async/http/protocol/http10.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. -require 'async/http/protocol/http10' -require 'async/http/a_protocol' +require "async/http/protocol/http10" +require "async/http/a_protocol" describe Async::HTTP::Protocol::HTTP10 do it_behaves_like Async::HTTP::AProtocol diff --git a/test/async/http/protocol/http11.rb b/test/async/http/protocol/http11.rb index c8bec4b..5b71f4b 100755 --- a/test/async/http/protocol/http11.rb +++ b/test/async/http/protocol/http11.rb @@ -7,13 +7,13 @@ # Copyright, 2023, by Josh Huber. # Copyright, 2024, by Anton Zhuravsky. -require 'async/http/protocol/http11' -require 'async/http/a_protocol' +require "async/http/protocol/http11" +require "async/http/a_protocol" describe Async::HTTP::Protocol::HTTP11 do it_behaves_like Async::HTTP::AProtocol - with '#as_json' do + with "#as_json" do include Sus::Fixtures::Async::HTTP::ServerContext let(:protocol) {subject} @@ -36,11 +36,11 @@ end end - with 'server' do + with "server" do include Sus::Fixtures::Async::HTTP::ServerContext let(:protocol) {subject} - with 'bad requests' do + with "bad requests" do def around current = Console.logger.level Console.logger.fatal! @@ -57,7 +57,7 @@ def around end end - with 'head request' do + with "head request" do let(:app) do Protocol::HTTP::Middleware.for do |request| Protocol::HTTP::Response[200, {}, ["Hello", "World"]] @@ -78,7 +78,7 @@ def around end end - with 'raw response' do + with "raw response" do let(:app) do Protocol::HTTP::Middleware.for do |request| peer = request.hijack! @@ -108,7 +108,7 @@ def around end end - with 'full hijack with empty response' do + with "full hijack with empty response" do let(:body) {::Protocol::HTTP::Body::Buffered.new([], 0)} let(:app) do diff --git a/test/async/http/protocol/http11/desync.rb b/test/async/http/protocol/http11/desync.rb index 4ee31c1..1f5f2ff 100644 --- a/test/async/http/protocol/http11/desync.rb +++ b/test/async/http/protocol/http11/desync.rb @@ -3,9 +3,9 @@ # Released under the MIT License. # Copyright, 2021-2024, by Samuel Williams. -require 'async/http/protocol/http11' +require "async/http/protocol/http11" -require 'sus/fixtures/async/http/server_context' +require "sus/fixtures/async/http/server_context" describe Async::HTTP::Protocol::HTTP11 do include Sus::Fixtures::Async::ReactorContext @@ -35,7 +35,7 @@ def around 100.times do tasks << task.async{ loop do - response = client.get('/a') + response = client.get("/a") expect(response.read).to be == "/a" rescue Exception => exception backtraces << exception&.backtrace @@ -49,7 +49,7 @@ def around 100.times do tasks << task.async{ loop do - response = client.get('/b') + response = client.get("/b") expect(response.read).to be == "/b" rescue Exception => exception backtraces << exception&.backtrace diff --git a/test/async/http/protocol/http2.rb b/test/async/http/protocol/http2.rb index ada1be0..bfef591 100644 --- a/test/async/http/protocol/http2.rb +++ b/test/async/http/protocol/http2.rb @@ -3,13 +3,13 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require 'async/http/protocol/http2' -require 'async/http/a_protocol' +require "async/http/protocol/http2" +require "async/http/a_protocol" describe Async::HTTP::Protocol::HTTP2 do it_behaves_like Async::HTTP::AProtocol - with '#as_json' do + with "#as_json" do include Sus::Fixtures::Async::HTTP::ServerContext let(:protocol) {subject} @@ -32,20 +32,20 @@ end end - with 'server' do + with "server" do include Sus::Fixtures::Async::HTTP::ServerContext let(:protocol) {subject} - with 'bad requests' do + with "bad requests" do it "should fail with explicit authority" do expect do - client.post("/", [[':authority', 'foo']]) + client.post("/", [[":authority", "foo"]]) end.to raise_exception(Protocol::HTTP2::StreamError) end end - with 'closed streams' do - it 'should delete stream after response stream is closed' do + with "closed streams" do + it "should delete stream after response stream is closed" do response = client.get("/") connection = response.connection @@ -55,7 +55,7 @@ end end - with 'host header' do + with "host header" do let(:app) do Protocol::HTTP::Middleware.for do |request| Protocol::HTTP::Response[200, request.headers, ["Authority: #{request.authority.inspect}"]] @@ -69,17 +69,17 @@ def make_client(endpoint, **options) end it "should not send :authority header if host header is present" do - response = client.post("/", [['host', 'foo']]) + response = client.post("/", [["host", "foo"]]) - expect(response.headers).to have_keys('host') - expect(response.headers['host']).to be == 'foo' + expect(response.headers).to have_keys("host") + expect(response.headers["host"]).to be == "foo" # TODO Should HTTP/2 respect host header? expect(response.read).to be == "Authority: nil" end end - with 'stopping requests' do + with "stopping requests" do let(:notification) {Async::Notification.new} let(:app) do diff --git a/test/async/http/proxy.rb b/test/async/http/proxy.rb index 0d493a3..ff4b78e 100644 --- a/test/async/http/proxy.rb +++ b/test/async/http/proxy.rb @@ -4,19 +4,19 @@ # Copyright, 2019-2024, by Samuel Williams. # Copyright, 2020, by Sam Shadwell. -require 'async' -require 'async/http/proxy' -require 'async/http/protocol' -require 'async/http/body/hijack' +require "async" +require "async/http/proxy" +require "async/http/protocol" +require "async/http/body/hijack" -require 'sus/fixtures/async/http' +require "sus/fixtures/async/http" AProxy = Sus::Shared("a proxy") do include Sus::Fixtures::Async::HTTP::ServerContext let(:protocol) {subject} - with '.proxied_endpoint' do + with ".proxied_endpoint" do it "can construct valid endpoint" do endpoint = Async::HTTP::Endpoint.parse("http://www.codeotaku.com") proxied_endpoint = client.proxied_endpoint(endpoint) @@ -25,7 +25,7 @@ end end - with '.proxied_client' do + with ".proxied_client" do it "can construct valid client" do endpoint = Async::HTTP::Endpoint.parse("http://www.codeotaku.com") proxied_client = client.proxied_client(endpoint) @@ -34,7 +34,7 @@ end end - with 'CONNECT' do + with "CONNECT" do let(:app) do Protocol::HTTP::Middleware.for do |request| Async::HTTP::Body::Hijack.response(request, 200, {}) do |stream| @@ -65,7 +65,7 @@ end end - with 'echo server' do + with "echo server" do let(:app) do Protocol::HTTP::Middleware.for do |request| expect(request.path).to be == "localhost:1" @@ -116,7 +116,7 @@ end end - with 'proxied client' do + with "proxied client" do let(:app) do Protocol::HTTP::Middleware.for do |request| expect(request.method).to be == "CONNECT" @@ -171,7 +171,7 @@ let(:authorization_lambda) { ->(request) {true} } - it 'can get insecure website' do + it "can get insecure website" do endpoint = Async::HTTP::Endpoint.parse("http://www.google.com") proxy_client = client.proxied_client(endpoint) @@ -188,7 +188,7 @@ expect(proxy_client.pool).to be(:empty?) end - it 'can get secure website' do + it "can get secure website" do endpoint = Async::HTTP::Endpoint.parse("https://www.google.com") proxy_client = client.proxied_client(endpoint) @@ -200,19 +200,19 @@ proxy_client.close end - with 'authorization header required' do + with "authorization header required" do let(:authorization_lambda) do - ->(request) {request.headers['proxy-authorization'] == 'supersecretpassword' } + ->(request) {request.headers["proxy-authorization"] == "supersecretpassword" } end - with 'request includes headers' do - let(:headers) { [['proxy-authorization', 'supersecretpassword']] } + with "request includes headers" do + let(:headers) { [["proxy-authorization", "supersecretpassword"]] } - it 'succeeds' do + it "succeeds" do endpoint = Async::HTTP::Endpoint.parse("https://www.google.com") proxy_client = client.proxied_client(endpoint, headers) - response = proxy_client.get('/search') + response = proxy_client.get("/search") expect(response).not.to be(:failure?) expect(response.read).not.to be(:empty?) @@ -221,14 +221,14 @@ end end - with 'request does not include headers' do - it 'does not succeed' do + with "request does not include headers" do + it "does not succeed" do endpoint = Async::HTTP::Endpoint.parse("https://www.google.com") proxy_client = client.proxied_client(endpoint) expect do # Why is this response not 407? Because the response should come from the proxied connection, but that connection failed to be established. Because of that, there is no response. If we respond here with 407, it would be indistinguisable from the remote server returning 407. That would be an odd case, but none-the-less a valid one. - response = proxy_client.get('/search') + response = proxy_client.get("/search") end.to raise_exception(Async::HTTP::Proxy::ConnectFailure) proxy_client.close diff --git a/test/async/http/retry.rb b/test/async/http/retry.rb index 20fb7dd..ff3ec01 100644 --- a/test/async/http/retry.rb +++ b/test/async/http/retry.rb @@ -1,14 +1,14 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2023, by Samuel Williams. +# Copyright, 2020-2024, by Samuel Williams. -require 'async/http/client' -require 'async/http/endpoint' +require "async/http/client" +require "async/http/endpoint" -require 'sus/fixtures/async/http' +require "sus/fixtures/async/http" -describe 'consistent retry behaviour' do +describe "consistent retry behaviour" do include Sus::Fixtures::Async::HTTP::ServerContext let(:delay) {0.1} @@ -24,7 +24,7 @@ def make_request(body) # This causes the first request to fail with "SocketError" which is retried: Async::Task.current.with_timeout(delay / 2.0, SocketError) do - return client.get('/', {}, body) + return client.get("/", {}, body) end end diff --git a/test/async/http/server.rb b/test/async/http/server.rb index 1067d8f..1525089 100644 --- a/test/async/http/server.rb +++ b/test/async/http/server.rb @@ -3,18 +3,18 @@ # Released under the MIT License. # Copyright, 2024, by Samuel Williams. -require 'async/http/server' -require 'async/http/endpoint' -require 'sus/fixtures/async' +require "async/http/server" +require "async/http/endpoint" +require "sus/fixtures/async" describe Async::HTTP::Server do include Sus::Fixtures::Async::ReactorContext - let(:endpoint) {Async::HTTP::Endpoint.parse('http://localhost:0')} + let(:endpoint) {Async::HTTP::Endpoint.parse("http://localhost:0")} let(:app) {Protocol::HTTP::Middleware::Okay} let(:server) {subject.new(app, endpoint)} - with '#run' do + with "#run" do it "runs the server" do task = server.run diff --git a/test/async/http/ssl.rb b/test/async/http/ssl.rb index 6751e63..2e163ef 100644 --- a/test/async/http/ssl.rb +++ b/test/async/http/ssl.rb @@ -3,13 +3,13 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require 'async/http/server' -require 'async/http/client' -require 'async/http/endpoint' +require "async/http/server" +require "async/http/client" +require "async/http/endpoint" -require 'sus/fixtures/async' -require 'sus/fixtures/openssl' -require 'sus/fixtures/async/http' +require "sus/fixtures/async" +require "sus/fixtures/openssl" +require "sus/fixtures/async/http" describe Async::HTTP::Server do include Sus::Fixtures::Async::HTTP::ServerContext diff --git a/test/async/http/statistics.rb b/test/async/http/statistics.rb index df0aa03..2b35a43 100644 --- a/test/async/http/statistics.rb +++ b/test/async/http/statistics.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2023, by Samuel Williams. +# Copyright, 2018-2024, by Samuel Williams. -require 'async/http/statistics' -require 'sus/fixtures/async/http' +require "async/http/statistics" +require "sus/fixtures/async/http" describe Async::HTTP::Statistics do include Sus::Fixtures::Async::HTTP::ServerContext diff --git a/test/rack/test.rb b/test/rack/test.rb index a1d5b66..3d04d4d 100644 --- a/test/rack/test.rb +++ b/test/rack/test.rb @@ -3,11 +3,11 @@ # Released under the MIT License. # Copyright, 2019-2024, by Samuel Williams. -require 'sus/fixtures/async' -require 'async/http' +require "sus/fixtures/async" +require "async/http" -require 'rack/test' -require 'rack/builder' +require "rack/test" +require "rack/builder" describe Rack::Test do include Sus::Fixtures::Async::ReactorContext From 3e9970ac60ada86206583a942d24063ab8aea214 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 20 Sep 2024 10:34:57 +1200 Subject: [PATCH 075/125] Don't hide errors during tests. --- config/sus.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sus.rb b/config/sus.rb index 13414a2..d0861d2 100644 --- a/config/sus.rb +++ b/config/sus.rb @@ -4,7 +4,7 @@ # Copyright, 2017-2024, by Samuel Williams. # Copyright, 2018, by Janko Marohnić. -ENV["CONSOLE_LEVEL"] ||= "fatal" +# ENV["CONSOLE_LEVEL"] ||= "fatal" require "covered/sus" include Covered::Sus From 3920950e9d008466b87cc24d04720cf2507f08da Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 20 Sep 2024 10:47:47 +1200 Subject: [PATCH 076/125] Update releases. --- releases.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/releases.md b/releases.md index 6bc17f8..070c8df 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,16 @@ # Releases +## Unreleased + + - Improved HTTP/1 connection handling. + - The input stream is no longer closed when the output stream is closed. + +## v0.76.0 + + - `Async::HTTP::Body::Writable` is moved to `Protocol::HTTP::Body::Writable`. + - Remove `Async::HTTP::Body::Delayed` with no replacement. + - Remove `Async::HTTP::Body::Slowloris` with no replacement. + ## v0.75.0 - Better handling of HTTP/1 \<-\> HTTP/2 proxying, specifically upgrade/CONNECT requests. From 2bf7b9fd935b6871c98fd4d4aea2b4a1126ab3f7 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 20 Sep 2024 10:47:56 +1200 Subject: [PATCH 077/125] Bump minor version. --- lib/async/http/version.rb | 2 +- readme.md | 11 +++++++++++ releases.md | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index e912314..1add107 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.76.0" + VERSION = "0.77.0" end end diff --git a/readme.md b/readme.md index 350f33a..4d2527e 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,17 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. +### v0.77.0 + + - Improved HTTP/1 connection handling. + - The input stream is no longer closed when the output stream is closed. + +### v0.76.0 + + - `Async::HTTP::Body::Writable` is moved to `Protocol::HTTP::Body::Writable`. + - Remove `Async::HTTP::Body::Delayed` with no replacement. + - Remove `Async::HTTP::Body::Slowloris` with no replacement. + ### v0.75.0 - Better handling of HTTP/1 \<-\> HTTP/2 proxying, specifically upgrade/CONNECT requests. diff --git a/releases.md b/releases.md index 070c8df..c95f4b8 100644 --- a/releases.md +++ b/releases.md @@ -1,6 +1,6 @@ # Releases -## Unreleased +## v0.77.0 - Improved HTTP/1 connection handling. - The input stream is no longer closed when the output stream is closed. From b5292d6fc01b307af3d21fe9080a074b291d798f Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 22 Sep 2024 13:04:21 +1200 Subject: [PATCH 078/125] Reduce test output. --- lib/async/http/protocol/request.rb | 4 ++++ lib/async/http/protocol/response.rb | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/lib/async/http/protocol/request.rb b/lib/async/http/protocol/request.rb index fabfb42..bada183 100644 --- a/lib/async/http/protocol/request.rb +++ b/lib/async/http/protocol/request.rb @@ -41,6 +41,10 @@ def remote_address def remote_address= value @remote_address = value end + + def inspect + "#<#{self.class}:0x#{self.object_id.to_s(16)} method=#{method} path=#{path} version=#{version}>" + end end end end diff --git a/lib/async/http/protocol/response.rb b/lib/async/http/protocol/response.rb index e55998c..751e8e9 100644 --- a/lib/async/http/protocol/response.rb +++ b/lib/async/http/protocol/response.rb @@ -33,6 +33,10 @@ def remote_address def remote_address= value @remote_address = value end + + def inspect + "#<#{self.class}:0x#{self.object_id.to_s(16)} status=#{status}>" + end end end end From 30f502cbeae60460af92ddebd085b843fc1fec7c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 22 Sep 2024 13:05:20 +1200 Subject: [PATCH 079/125] Better handling of state transitions. --- lib/async/http/body/finishable.rb | 56 -------------------- lib/async/http/client.rb | 1 - lib/async/http/protocol/http1/finishable.rb | 58 +++++++++++++++++++++ lib/async/http/protocol/http1/server.rb | 25 +++++++-- 4 files changed, 78 insertions(+), 62 deletions(-) delete mode 100644 lib/async/http/body/finishable.rb create mode 100644 lib/async/http/protocol/http1/finishable.rb diff --git a/lib/async/http/body/finishable.rb b/lib/async/http/body/finishable.rb deleted file mode 100644 index 38de56c..0000000 --- a/lib/async/http/body/finishable.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -# Released under the MIT License. -# Copyright, 2024, by Samuel Williams. - -require "protocol/http/body/wrapper" -require "async/variable" - -module Async - module HTTP - module Body - # Keeps track of whether a body is being read, and if so, waits for it to be closed. - class Finishable < ::Protocol::HTTP::Body::Wrapper - def initialize(body) - super(body) - - @closed = Async::Variable.new - @error = nil - - @reading = false - end - - def reading? - @reading - end - - def read - @reading = true - - super - end - - def close(error = nil) - unless @closed.resolved? - @error = error - @closed.value = true - end - - super - end - - def wait - if @reading - @closed.wait - else - self.discard - end - end - - def inspect - "#<#{self.class} closed=#{@closed} error=#{@error}> | #{super}" - end - end - end - end -end diff --git a/lib/async/http/client.rb b/lib/async/http/client.rb index b1ab9c9..d4fdfc5 100755 --- a/lib/async/http/client.rb +++ b/lib/async/http/client.rb @@ -14,7 +14,6 @@ require "traces/provider" require_relative "protocol" -require_relative "body/finishable" module Async module HTTP diff --git a/lib/async/http/protocol/http1/finishable.rb b/lib/async/http/protocol/http1/finishable.rb new file mode 100644 index 0000000..9b2195c --- /dev/null +++ b/lib/async/http/protocol/http1/finishable.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2024, by Samuel Williams. + +require "protocol/http/body/wrapper" +require "async/variable" + +module Async + module HTTP + module Protocol + module HTTP1 + # Keeps track of whether a body is being read, and if so, waits for it to be closed. + class Finishable < ::Protocol::HTTP::Body::Wrapper + def initialize(body) + super(body) + + @closed = Async::Variable.new + @error = nil + + @reading = false + end + + def reading? + @reading + end + + def read + @reading = true + + super + end + + def close(error = nil) + unless @closed.resolved? + @error = error + @closed.value = true + end + + super + end + + def wait + if @reading + @closed.wait + else + self.discard + end + end + + def inspect + "#<#{self.class} closed=#{@closed} error=#{@error}> | #{super}" + end + end + end + end + end +end diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index fd333c8..d75337c 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -7,7 +7,7 @@ # Copyright, 2024, by Anton Zhuravsky. require_relative "connection" -require_relative "../../body/finishable" +require_relative "finishable" require "console/event/failure" @@ -16,6 +16,18 @@ module HTTP module Protocol module HTTP1 class Server < Connection + def initialize(...) + super + + @ready = Async::Notification.new + end + + def closed! + super + + @ready.signal + end + def fail_request(status) @persistent = false write_response(@version, status, {}) @@ -26,6 +38,11 @@ def fail_request(status) end def next_request + # Wait for the connection to become idle before reading the next request: + unless idle? + @ready.wait + end + # The default is true. return unless @persistent @@ -49,7 +66,7 @@ def each(task: Task.current) while request = next_request if body = request.body - finishable = Body::Finishable.new(body) + finishable = Finishable.new(body) request.body = finishable end @@ -126,10 +143,8 @@ def each(task: Task.current) request&.finish end + # Discard or wait for the input body to be consumed: finishable&.wait - - # This ensures we yield at least once every iteration of the loop and allow other fibers to execute. - task.yield rescue => error raise ensure From 87a2a7297d15aecba7a65790c43ed1d0fb02e7da Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 22 Sep 2024 13:06:15 +1200 Subject: [PATCH 080/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 1add107..f0b00d9 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.77.0" + VERSION = "0.78.0" end end From 6d6fb1ab43be75fd8003d0a22e4ef7d166031f83 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 23 Sep 2024 12:26:01 +1200 Subject: [PATCH 081/125] Wait for input to finish even when streaming, if necessary. --- lib/async/http/protocol/http1/finishable.rb | 8 ++++++-- lib/async/http/protocol/http1/server.rb | 15 +++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/async/http/protocol/http1/finishable.rb b/lib/async/http/protocol/http1/finishable.rb index 9b2195c..5c972e5 100644 --- a/lib/async/http/protocol/http1/finishable.rb +++ b/lib/async/http/protocol/http1/finishable.rb @@ -40,11 +40,15 @@ def close(error = nil) super end - def wait + def wait(persistent = true) if @reading @closed.wait - else + elsif persistent + # If the connection can be reused, let's gracefully discard the body: self.discard + else + # Else, we don't care about the body, so we can close it immediately: + self.close end end diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index d75337c..e4379dd 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -96,8 +96,8 @@ def each(task: Task.current) request = nil response = nil - # We must return here as no further request processing can be done: - return body.call(stream) + # In the case of streaming, `finishable` should wrap a `Remainder` body, which we can safely discard later on. + body.call(stream) elsif response.status == 101 # This code path is to support legacy behavior where the response status is set to 101, but the protocol is not upgraded. This may not be a valid use case, but it is supported for compatibility. We expect the response headers to contain the `upgrade` header. write_response(@version, response.status, response.headers) @@ -108,8 +108,7 @@ def each(task: Task.current) request = nil response = nil - # We must return here as no further request processing can be done: - return body&.call(stream) + body&.call(stream) else write_response(@version, response.status, response.headers) @@ -143,8 +142,12 @@ def each(task: Task.current) request&.finish end - # Discard or wait for the input body to be consumed: - finishable&.wait + if finishable + finishable.wait(@persistent) + else + # Do not remove this line or you will unleash the gods of concurrency hell. + task.yield + end rescue => error raise ensure From f465183676e21f6ac54ca99c7bc06a376a2a8d23 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 24 Sep 2024 15:37:25 +1200 Subject: [PATCH 082/125] Better HTTP/1 connection handling. --- async-http.gemspec | 4 +-- lib/async/http/protocol/http1/client.rb | 21 ++++++------ lib/async/http/protocol/http1/connection.rb | 2 ++ lib/async/http/protocol/http1/finishable.rb | 4 +-- lib/async/http/protocol/http1/server.rb | 36 +++++++++++---------- lib/async/http/protocol/http2/stream.rb | 14 ++++---- test/async/http/proxy.rb | 2 -- 7 files changed, 45 insertions(+), 38 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index af9026d..e19643e 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -29,7 +29,7 @@ Gem::Specification.new do |spec| spec.add_dependency "io-endpoint", "~> 0.11" spec.add_dependency "io-stream", "~> 0.4" spec.add_dependency "protocol-http", "~> 0.37" - spec.add_dependency "protocol-http1", "~> 0.25" - spec.add_dependency "protocol-http2", "~> 0.18" + spec.add_dependency "protocol-http1", "~> 0.27" + spec.add_dependency "protocol-http2", "~> 0.19" spec.add_dependency "traces", ">= 0.10" end diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index ec295f2..b780325 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -18,11 +18,12 @@ def initialize(...) attr_accessor :pool - def closed! + def closed(error = nil) super if pool = @pool @pool = nil + # If the connection is not reusable, this will retire it from the connection pool and invoke `#close`. pool.release(self) end end @@ -50,30 +51,32 @@ def call(request, task: Task.current) task.async(annotation: "Upgrading request...") do # If this fails, this connection will be closed. write_upgrade_body(protocol, body) + rescue => error + self.close(error) end elsif request.connect? task.async(annotation: "Tunnneling request...") do write_tunnel_body(@version, body) + rescue => error + self.close(error) end else task.async(annotation: "Streaming request...") do # Once we start writing the body, we can't recover if the request fails. That's because the body might be generated dynamically, streaming, etc. write_body(@version, body, false, trailer) + rescue => error + self.close(error) end end elsif protocol = request.protocol write_upgrade_body(protocol) else - write_body(@version, body, false, trailer) + write_body(@version, request.body, false, trailer) end - response = Response.read(self, request) - - return response - rescue - # This will ensure that #reusable? returns false. - self.close - + return Response.read(self, request) + rescue => error + self.close(error) raise end end diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index bfaf0d7..46aa99e 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -43,6 +43,8 @@ def http2? def read_line? @stream.read_until(CRLF) + rescue Errno::ECONNRESET + return nil end def read_line diff --git a/lib/async/http/protocol/http1/finishable.rb b/lib/async/http/protocol/http1/finishable.rb index 5c972e5..26aee3f 100644 --- a/lib/async/http/protocol/http1/finishable.rb +++ b/lib/async/http/protocol/http1/finishable.rb @@ -32,12 +32,12 @@ def read end def close(error = nil) + super + unless @closed.resolved? @error = error @closed.value = true end - - super end def wait(persistent = true) diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index e4379dd..bec8dd0 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -22,7 +22,7 @@ def initialize(...) @ready = Async::Notification.new end - def closed! + def closed(error = nil) super @ready.signal @@ -38,14 +38,12 @@ def fail_request(status) end def next_request - # Wait for the connection to become idle before reading the next request: - unless idle? + if closed? + return nil + elsif !idle? @ready.wait end - # The default is true. - return unless @persistent - # Read an incoming request: return unless request = Request.read(self) @@ -90,37 +88,41 @@ def each(task: Task.current) # We force a 101 response if the protocol is upgraded - HTTP/2 CONNECT will return 200 for success, but this won't be understood by HTTP/1 clients: write_response(@version, 101, response.headers) - stream = write_upgrade_body(protocol) - # At this point, the request body is hijacked, so we don't want to call #finish below. request = nil response = nil - # In the case of streaming, `finishable` should wrap a `Remainder` body, which we can safely discard later on. - body.call(stream) + if body.stream? + return body.call(write_upgrade_body(protocol)) + else + write_upgrade_body(protocol, body) + end elsif response.status == 101 # This code path is to support legacy behavior where the response status is set to 101, but the protocol is not upgraded. This may not be a valid use case, but it is supported for compatibility. We expect the response headers to contain the `upgrade` header. write_response(@version, response.status, response.headers) - stream = write_tunnel_body(version) - # Same as above: request = nil response = nil - body&.call(stream) + if body.stream? + return body.call(write_tunnel_body(version)) + else + write_tunnel_body(version, body) + end else write_response(@version, response.status, response.headers) if request.connect? and response.success? - stream = write_tunnel_body(version) - # Same as above: request = nil response = nil - # We must return here as no further request processing can be done: - return body.call(stream) + if body.stream? + return body.call(write_tunnel_body(version)) + else + write_tunnel_body(version, body) + end else head = request.head? diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index 505196d..2544e3b 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -62,9 +62,9 @@ def process_headers(frame) end # TODO this might need to be in an ensure block: - if @input and frame.end_stream? - @input.close_write + if input = @input and frame.end_stream? @input = nil + input.close_write end rescue ::Protocol::HTTP2::HeaderError => error Console.logger.debug(self, error) @@ -123,6 +123,8 @@ def send_body(body, trailer = nil) # Called when the output terminates normally. def finish_output(error = nil) + return if self.closed? + trailer = @output&.trailer @output = nil @@ -152,14 +154,14 @@ def window_updated(size) def closed(error) super - if @input - @input.close_write(error) + if input = @input @input = nil + input.close_write(error) end - if @output - @output.stop(error) + if output = @output @output = nil + output.stop(error) end if pool = @pool and @connection diff --git a/test/async/http/proxy.rb b/test/async/http/proxy.rb index ff4b78e..fe8d180 100644 --- a/test/async/http/proxy.rb +++ b/test/async/http/proxy.rb @@ -153,8 +153,6 @@ upstream.write(chunk) upstream.flush end - rescue Async::Wrapper::Cancelled - #ignore ensure Console.logger.debug(self) {"Finished writing to upstream..."} upstream.close_write From b9e1b10d8c059af15550cb08199766eb5ebc0777 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 24 Sep 2024 15:46:03 +1200 Subject: [PATCH 083/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index f0b00d9..d347126 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.78.0" + VERSION = "0.79.0" end end From 8a56b35071a876843b9e94c8517c78de50ab91ee Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 2 Oct 2024 21:13:32 +1300 Subject: [PATCH 084/125] Better window update synchronization. --- lib/async/http/protocol/http2/output.rb | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/async/http/protocol/http2/output.rb b/lib/async/http/protocol/http2/output.rb index 2f2efd4..32ec3d8 100644 --- a/lib/async/http/protocol/http2/output.rb +++ b/lib/async/http/protocol/http2/output.rb @@ -17,7 +17,8 @@ def initialize(stream, body, trailer = nil) @task = nil - @window_updated = Async::Condition.new + @guard = ::Mutex.new + @window_updated = ::ConditionVariable.new end attr :trailer @@ -33,17 +34,26 @@ def start(parent: Task.current) end def window_updated(size) - @window_updated.signal + @guard.synchronize do + @window_updated.signal + end end def write(chunk) until chunk.empty? maximum_size = @stream.available_frame_size - while maximum_size <= 0 - @window_updated.wait - - maximum_size = @stream.available_frame_size + # We try to avoid synchronization if possible: + if maximum_size <= 0 + @guard.synchronize do + maximum_size = @stream.available_frame_size + + while maximum_size <= 0 + @window_updated.wait(@guard) + + maximum_size = @stream.available_frame_size + end + end end break unless chunk = send_data(chunk, maximum_size) From b185db31f59b9669eeb5d5ecd3ee0e24e2d22c88 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 2 Oct 2024 22:29:26 +1300 Subject: [PATCH 085/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index d347126..2a8af40 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.79.0" + VERSION = "0.80.0" end end From 5484d1eceb21ce393298e026c7857a9ee2caabc7 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 12 Oct 2024 10:13:40 +1300 Subject: [PATCH 086/125] Explicitly require URI schemes. --- lib/async/http/endpoint.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index 614f1b0..e8579bf 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -14,6 +14,11 @@ require_relative "protocol/http" require_relative "protocol/https" +require "uri" + +# Compatibility with Ruby 3.1.2 +require "uri/wss" + module Async module HTTP # Represents a way to connect to a remote HTTP server. From ba0f7987476e31b3e90482d5efb6b6eda0546e87 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 12 Oct 2024 12:47:30 +1300 Subject: [PATCH 087/125] `read_line?` should return `nil` on timeout and any other error, basically. --- lib/async/http/protocol/http1/connection.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index 46aa99e..1515151 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -43,7 +43,8 @@ def http2? def read_line? @stream.read_until(CRLF) - rescue Errno::ECONNRESET + rescue => error + # Bascially, any error will cause the connection to be closed: return nil end From 6e6f087162ba6644303bdb1e866b0a30cb133c79 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 12 Oct 2024 13:13:57 +1300 Subject: [PATCH 088/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 2a8af40..58b3158 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.80.0" + VERSION = "0.80.1" end end From 0eb1c3df50b9c3d9e38ce6a78ff62f9caf34b47f Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 12 Oct 2024 19:22:39 +1300 Subject: [PATCH 089/125] Fix readme formatting. --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 4d2527e..421335c 100644 --- a/readme.md +++ b/readme.md @@ -29,7 +29,7 @@ Please see the [project releases](https://socketry.github.io/async-http/releases ### v0.75.0 - - Better handling of HTTP/1 \<-\> HTTP/2 proxying, specifically upgrade/CONNECT requests. + - Better handling of HTTP/1 \<-\> HTTP/2 proxying, specifically upgrade/CONNECT requests. ### v0.74.0 From deafa5d05a1335e544f28cbeff0e6584f7310489 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 13 Oct 2024 00:30:11 +1300 Subject: [PATCH 090/125] Add support for instrumentation tags. --- async-http.gemspec | 7 ++++--- lib/async/http/client.rb | 21 ++++++++++++++++----- releases.md | 4 ++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index e19643e..84f6147 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -25,11 +25,12 @@ Gem::Specification.new do |spec| spec.required_ruby_version = ">= 3.1" spec.add_dependency "async", ">= 2.10.2" - spec.add_dependency "async-pool", "~> 0.7" - spec.add_dependency "io-endpoint", "~> 0.11" + spec.add_dependency "async-pool", "~> 0.9" + spec.add_dependency "io-endpoint", "~> 0.14" spec.add_dependency "io-stream", "~> 0.4" spec.add_dependency "protocol-http", "~> 0.37" spec.add_dependency "protocol-http1", "~> 0.27" spec.add_dependency "protocol-http2", "~> 0.19" - spec.add_dependency "traces", ">= 0.10" + spec.add_dependency "traces", "~> 0.10" + spec.add_dependency "metrics", "~> 0.12" end diff --git a/lib/async/http/client.rb b/lib/async/http/client.rb index d4fdfc5..c755e87 100755 --- a/lib/async/http/client.rb +++ b/lib/async/http/client.rb @@ -18,7 +18,6 @@ module Async module HTTP DEFAULT_RETRIES = 3 - DEFAULT_CONNECTION_LIMIT = nil class Client < ::Protocol::HTTP::Methods # Provides a robust interface to a server. @@ -30,12 +29,12 @@ class Client < ::Protocol::HTTP::Methods # @param protocol [Protocol::HTTP1 | Protocol::HTTP2 | Protocol::HTTPS] the protocol to use. # @param scheme [String] The default scheme to set to requests. # @param authority [String] The default authority to set to requests. - def initialize(endpoint, protocol: endpoint.protocol, scheme: endpoint.scheme, authority: endpoint.authority, retries: DEFAULT_RETRIES, connection_limit: DEFAULT_CONNECTION_LIMIT) + def initialize(endpoint, protocol: endpoint.protocol, scheme: endpoint.scheme, authority: endpoint.authority, retries: DEFAULT_RETRIES, **options) @endpoint = endpoint @protocol = protocol @retries = retries - @pool = make_pool(connection_limit) + @pool = make_pool(**options) @scheme = scheme @authority = authority @@ -191,8 +190,20 @@ def make_response(request, connection) return response end - def make_pool(connection_limit) - Async::Pool::Controller.wrap(limit: connection_limit) do + def assign_default_tags(tags) + tags[:endpoint] = @endpoint.to_s + tags[:protocol] = @protocol.to_s + end + + def make_pool(**options) + if connection_limit = options.delete(:connection_limit) + warn "The connection_limit: option is deprecated, please use limit: instead.", uplevel: 2 + options[:limit] = connection_limit + end + + self.assign_default_tags(options[:tags] ||= {}) + + Async::Pool::Controller.wrap(**options) do Console.logger.debug(self) {"Making connection to #{@endpoint.inspect}"} @protocol.client(@endpoint.connect) diff --git a/releases.md b/releases.md index c95f4b8..149eca3 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - Expose `protocol` and `endpoint` as tags to `async-pool` for improved instrumentation. + ## v0.77.0 - Improved HTTP/1 connection handling. From 84a70162e8243105ec6ffa1581d471408a03962d Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 13 Oct 2024 15:15:56 +1300 Subject: [PATCH 091/125] Bump minor version. --- lib/async/http/version.rb | 2 +- readme.md | 4 ++++ releases.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 58b3158..274d03e 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.80.1" + VERSION = "0.81.0" end end diff --git a/readme.md b/readme.md index 421335c..cae808e 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,10 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. +### v0.81.0 + + - Expose `protocol` and `endpoint` as tags to `async-pool` for improved instrumentation. + ### v0.77.0 - Improved HTTP/1 connection handling. diff --git a/releases.md b/releases.md index 149eca3..cd7e8a9 100644 --- a/releases.md +++ b/releases.md @@ -1,6 +1,6 @@ # Releases -## Unreleased +## v0.81.0 - Expose `protocol` and `endpoint` as tags to `async-pool` for improved instrumentation. From 24f45d16c0fd183b25a554ecfe1a5108bd7acc10 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 16 Oct 2024 01:13:51 +1300 Subject: [PATCH 092/125] Use `read_line` implementation from `protocol-http1`. --- async-http.gemspec | 4 ++-- lib/async/http/protocol/http1/connection.rb | 11 ----------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index 84f6147..896fc64 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -27,9 +27,9 @@ Gem::Specification.new do |spec| spec.add_dependency "async", ">= 2.10.2" spec.add_dependency "async-pool", "~> 0.9" spec.add_dependency "io-endpoint", "~> 0.14" - spec.add_dependency "io-stream", "~> 0.4" + spec.add_dependency "io-stream", "~> 0.6" spec.add_dependency "protocol-http", "~> 0.37" - spec.add_dependency "protocol-http1", "~> 0.27" + spec.add_dependency "protocol-http1", ">= 0.28.1" spec.add_dependency "protocol-http2", "~> 0.19" spec.add_dependency "traces", "~> 0.10" spec.add_dependency "metrics", "~> 0.12" diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index 1515151..fb41351 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -41,17 +41,6 @@ def http2? false end - def read_line? - @stream.read_until(CRLF) - rescue => error - # Bascially, any error will cause the connection to be closed: - return nil - end - - def read_line - @stream.read_until(CRLF) or raise EOFError, "Could not read line!" - end - def peer @stream.io end From f2b31fb0d5b3bc4bb35e66e74e33fc1e050f0a31 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 16 Oct 2024 01:47:39 +1300 Subject: [PATCH 093/125] Update release notes. --- releases.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/releases.md b/releases.md index cd7e8a9..6a11bdb 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - `protocol-http1` introduces a line length limit for request line, response line, header lines and chunk length lines. + ## v0.81.0 - Expose `protocol` and `endpoint` as tags to `async-pool` for improved instrumentation. From 0c00924936a5ef0042a7994f55882037bd97706a Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 16 Oct 2024 01:47:46 +1300 Subject: [PATCH 094/125] Bump minor version. --- lib/async/http/version.rb | 2 +- readme.md | 4 ++++ releases.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 274d03e..9d69fe9 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.81.0" + VERSION = "0.82.0" end end diff --git a/readme.md b/readme.md index cae808e..c574500 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,10 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. +### v0.82.0 + + - `protocol-http1` introduces a line length limit for request line, response line, header lines and chunk length lines. + ### v0.81.0 - Expose `protocol` and `endpoint` as tags to `async-pool` for improved instrumentation. diff --git a/releases.md b/releases.md index 6a11bdb..cf5c1b8 100644 --- a/releases.md +++ b/releases.md @@ -1,6 +1,6 @@ # Releases -## Unreleased +## v0.82.0 - `protocol-http1` introduces a line length limit for request line, response line, header lines and chunk length lines. From 710d883a2d98f2694d4e3f9db7e00732abc34c98 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 16 Oct 2024 20:48:28 +1300 Subject: [PATCH 095/125] Don't set `@input` to `nil` until the stream is closed. `wait_for_input` may race on a end frame. --- lib/async/http/protocol/http2/stream.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index 2544e3b..cf50fdb 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -61,10 +61,8 @@ def process_headers(frame) self.receive_initial_headers(super, frame.end_stream?) end - # TODO this might need to be in an ensure block: - if input = @input and frame.end_stream? - @input = nil - input.close_write + if @input and frame.end_stream? + @input.close_write end rescue ::Protocol::HTTP2::HeaderError => error Console.logger.debug(self, error) @@ -103,7 +101,6 @@ def process_data(frame) if frame.end_stream? @input.close_write - @input = nil end end From e0da7ab4841ecfb6064cc321d6bb82b6e883ccc4 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 16 Oct 2024 20:52:23 +1300 Subject: [PATCH 096/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 9d69fe9..4b6e80e 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.82.0" + VERSION = "0.82.1" end end From 1486ed2e4ba9c1c8668defec86d611904615dfe8 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 22 Oct 2024 16:33:01 +1300 Subject: [PATCH 097/125] Add tracing around `Client#make_response`. --- lib/async/http/client.rb | 66 +++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/lib/async/http/client.rb b/lib/async/http/client.rb index c755e87..73c71de 100755 --- a/lib/async/http/client.rb +++ b/lib/async/http/client.rb @@ -101,7 +101,7 @@ def call(request) # As we cache pool, it's possible these pool go bad (e.g. closed by remote host). In this case, we need to try again. It's up to the caller to impose a timeout on this. If this is the last attempt, we force a new connection. connection = @pool.acquire - response = make_response(request, connection) + response = make_response(request, connection, attempt) # This signals that the ensure block below should not try to release the connection, because it's bound into the response which will be returned: connection = nil @@ -140,6 +140,36 @@ def inspect "#<#{self.class} authority=#{@authority.inspect}>" end + protected + + def make_response(request, connection, attempt) + response = request.call(connection) + + response.pool = @pool + + return response + end + + def assign_default_tags(tags) + tags[:endpoint] = @endpoint.to_s + tags[:protocol] = @protocol.to_s + end + + def make_pool(**options) + if connection_limit = options.delete(:connection_limit) + warn "The connection_limit: option is deprecated, please use limit: instead.", uplevel: 2 + options[:limit] = connection_limit + end + + self.assign_default_tags(options[:tags] ||= {}) + + Async::Pool::Controller.wrap(**options) do + Console.logger.debug(self) {"Making connection to #{@endpoint.inspect}"} + + @protocol.client(@endpoint.connect) + end + end + Traces::Provider(self) do def call(request) attributes = { @@ -178,35 +208,15 @@ def call(request) end end end - end - - protected - - def make_response(request, connection) - response = request.call(connection) - - response.pool = @pool - return response - end - - def assign_default_tags(tags) - tags[:endpoint] = @endpoint.to_s - tags[:protocol] = @protocol.to_s - end - - def make_pool(**options) - if connection_limit = options.delete(:connection_limit) - warn "The connection_limit: option is deprecated, please use limit: instead.", uplevel: 2 - options[:limit] = connection_limit - end - - self.assign_default_tags(options[:tags] ||= {}) - - Async::Pool::Controller.wrap(**options) do - Console.logger.debug(self) {"Making connection to #{@endpoint.inspect}"} + def make_response(request, connection, attempt) + attributes = { + attempt: attempt, + } - @protocol.client(@endpoint.connect) + Traces.trace("async.http.client.make_response", attributes: attributes) do + super + end end end end From 6041cac341fd9345e53e36783edcf297ca3433d1 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 22 Oct 2024 19:56:12 +1300 Subject: [PATCH 098/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 4b6e80e..da693e4 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.82.1" + VERSION = "0.82.2" end end From c6307cf611ee650caf48d04f9ba64d55c480ee76 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 3 Nov 2024 23:04:46 +1300 Subject: [PATCH 099/125] Add client request tracing. --- lib/async/http/protocol/http1/client.rb | 16 ++++++++++++++++ lib/async/http/protocol/http2/client.rb | 25 ++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index b780325..fd478e5 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -5,6 +5,8 @@ require_relative "connection" +require "traces/provider" + module Async module HTTP module Protocol @@ -79,6 +81,20 @@ def call(request, task: Task.current) self.close(error) raise end + + Traces::Provider(self) do + def write_request(...) + Traces.trace("async.http.protocol.http1.client.write_request") do + super + end + end + + def read_response(...) + Traces.trace("async.http.protocol.http1.client.read_response") do + super + end + end + end end end end diff --git a/lib/async/http/protocol/http2/client.rb b/lib/async/http/protocol/http2/client.rb index ba6e979..5ad422f 100644 --- a/lib/async/http/protocol/http2/client.rb +++ b/lib/async/http/protocol/http2/client.rb @@ -6,6 +6,7 @@ require_relative "connection" require_relative "response" +require "traces/provider" require "protocol/http2/client" module Async @@ -34,10 +35,32 @@ def call(request) @count += 1 response = create_response + write_request(response, request) + read_response(response) + + return response + end + + def write_request(response, request) response.send_request(request) + end + + def read_response(response) response.wait + end + + Traces::Provider(self) do + def write_request(...) + Traces.trace("async.http.protocol.http2.client.write_request") do + super + end + end - return response + def read_response(...) + Traces.trace("async.http.protocol.http2.client.read_response") do + super + end + end end end end From 6429f222cb9ffdfbbf3e3a6fe805c4d4768995bf Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 3 Nov 2024 23:39:36 +1300 Subject: [PATCH 100/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index da693e4..f3f2a8f 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.82.2" + VERSION = "0.82.3" end end From 67bf1cf624ffc335d43d9be856b0ead60f54b084 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 9 Nov 2024 22:08:53 +1300 Subject: [PATCH 101/125] Expose cached peer address interface. (#189) --- lib/async/http/protocol/http1/connection.rb | 3 +- lib/async/http/protocol/http2/connection.rb | 3 +- lib/async/http/protocol/peer.rb | 32 +++++++++++++++++++++ lib/async/http/protocol/request.rb | 10 ++----- lib/async/http/protocol/response.rb | 10 ++----- lib/async/http/server.rb | 3 -- 6 files changed, 40 insertions(+), 21 deletions(-) create mode 100644 lib/async/http/protocol/peer.rb diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index fb41351..f008647 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -5,6 +5,7 @@ require "protocol/http1" +require_relative "../peer" require_relative "request" require_relative "response" @@ -42,7 +43,7 @@ def http2? end def peer - @stream.io + @peer ||= Peer.for(@stream.io) end attr :count diff --git a/lib/async/http/protocol/http2/connection.rb b/lib/async/http/protocol/http2/connection.rb index 7d989d0..1a04e9c 100644 --- a/lib/async/http/protocol/http2/connection.rb +++ b/lib/async/http/protocol/http2/connection.rb @@ -5,6 +5,7 @@ # Copyright, 2020, by Bruno Sutic. require_relative "stream" +require_relative "../peer" require "async/semaphore" @@ -112,7 +113,7 @@ def read_in_background(parent: Task.current) attr :promises def peer - @stream.io + @peer ||= Peer.for(@stream.io) end attr :count diff --git a/lib/async/http/protocol/peer.rb b/lib/async/http/protocol/peer.rb new file mode 100644 index 0000000..4331b0d --- /dev/null +++ b/lib/async/http/protocol/peer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2017-2024, by Samuel Williams. + +module Async + module HTTP + module Protocol + # Provide a well defined, cached representation of a peer (address). + class Peer + def self.for(io) + if address = io.remote_address + return new(address) + end + end + + def initialize(address) + @address = address + + if address.ip? + @ip_address = @address.ip_address + end + end + + attr :address + attr :ip_address + + alias remote_address address + end + end + end +end diff --git a/lib/async/http/protocol/request.rb b/lib/async/http/protocol/request.rb index bada183..b364de0 100644 --- a/lib/async/http/protocol/request.rb +++ b/lib/async/http/protocol/request.rb @@ -29,17 +29,11 @@ def write_interim_response(status, headers = nil) end def peer - if connection = self.connection - connection.peer - end + self.connection&.peer end def remote_address - @remote_address ||= peer.remote_address - end - - def remote_address= value - @remote_address = value + self.peer&.address end def inspect diff --git a/lib/async/http/protocol/response.rb b/lib/async/http/protocol/response.rb index 751e8e9..76f761b 100644 --- a/lib/async/http/protocol/response.rb +++ b/lib/async/http/protocol/response.rb @@ -21,17 +21,11 @@ def hijack? end def peer - if connection = self.connection - connection.peer - end + self.connection&.peer end def remote_address - @remote_address ||= peer.remote_address - end - - def remote_address= value - @remote_address = value + self.peer&.remote_address end def inspect diff --git a/lib/async/http/server.rb b/lib/async/http/server.rb index bb7c515..be0a536 100755 --- a/lib/async/http/server.rb +++ b/lib/async/http/server.rb @@ -52,9 +52,6 @@ def accept(peer, address, task: Task.current) # https://tools.ietf.org/html/rfc7230#section-5.5 request.scheme ||= self.scheme - # This is a slight optimization to avoid having to get the address from the socket. - request.remote_address = address - # Console.logger.debug(self) {"Incoming request from #{address.inspect}: #{request.method} #{request.path}"} # If this returns nil, we assume that the connection has been hijacked. From ebbd1555ad0982d1520f9bbbb905303b4ab598dd Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 10 Nov 2024 00:43:50 +1300 Subject: [PATCH 102/125] move `Peer` to `protocol-http` gem. --- async-http.gemspec | 2 +- lib/async/http/protocol/http1/connection.rb | 8 +++--- lib/async/http/protocol/http2/connection.rb | 4 +-- lib/async/http/protocol/peer.rb | 32 --------------------- 4 files changed, 7 insertions(+), 39 deletions(-) delete mode 100644 lib/async/http/protocol/peer.rb diff --git a/async-http.gemspec b/async-http.gemspec index 896fc64..b711ffa 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.9" spec.add_dependency "io-endpoint", "~> 0.14" spec.add_dependency "io-stream", "~> 0.6" - spec.add_dependency "protocol-http", "~> 0.37" + spec.add_dependency "protocol-http", "~> 0.43" spec.add_dependency "protocol-http1", ">= 0.28.1" spec.add_dependency "protocol-http2", "~> 0.19" spec.add_dependency "traces", "~> 0.10" diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index f008647..9f84364 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -3,12 +3,12 @@ # Released under the MIT License. # Copyright, 2018-2024, by Samuel Williams. -require "protocol/http1" - -require_relative "../peer" require_relative "request" require_relative "response" +require "protocol/http1" +require "protocol/http/peer" + module Async module HTTP module Protocol @@ -43,7 +43,7 @@ def http2? end def peer - @peer ||= Peer.for(@stream.io) + @peer ||= Protocol::HTTP::Peer.for(@stream.io) end attr :count diff --git a/lib/async/http/protocol/http2/connection.rb b/lib/async/http/protocol/http2/connection.rb index 1a04e9c..cfde0eb 100644 --- a/lib/async/http/protocol/http2/connection.rb +++ b/lib/async/http/protocol/http2/connection.rb @@ -5,8 +5,8 @@ # Copyright, 2020, by Bruno Sutic. require_relative "stream" -require_relative "../peer" +require "protocol/http/peer" require "async/semaphore" module Async @@ -113,7 +113,7 @@ def read_in_background(parent: Task.current) attr :promises def peer - @peer ||= Peer.for(@stream.io) + @peer ||= Protocol::HTTP::Peer.for(@stream.io) end attr :count diff --git a/lib/async/http/protocol/peer.rb b/lib/async/http/protocol/peer.rb deleted file mode 100644 index 4331b0d..0000000 --- a/lib/async/http/protocol/peer.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -# Released under the MIT License. -# Copyright, 2017-2024, by Samuel Williams. - -module Async - module HTTP - module Protocol - # Provide a well defined, cached representation of a peer (address). - class Peer - def self.for(io) - if address = io.remote_address - return new(address) - end - end - - def initialize(address) - @address = address - - if address.ip? - @ip_address = @address.ip_address - end - end - - attr :address - attr :ip_address - - alias remote_address address - end - end - end -end From 8ed0b5637d55c231781dde255ae2a404a9e8f132 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 10 Nov 2024 01:13:31 +1300 Subject: [PATCH 103/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index f3f2a8f..6fe9ed5 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.82.3" + VERSION = "0.83.0" end end From e9e389f95a8263a38f322bffdcdf21b9f7cf192d Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 10 Nov 2024 01:19:10 +1300 Subject: [PATCH 104/125] Fix constant resolution. --- lib/async/http/protocol/http1/connection.rb | 2 +- lib/async/http/protocol/http2/connection.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index 9f84364..1ceed5d 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -43,7 +43,7 @@ def http2? end def peer - @peer ||= Protocol::HTTP::Peer.for(@stream.io) + @peer ||= ::Protocol::HTTP::Peer.for(@stream.io) end attr :count diff --git a/lib/async/http/protocol/http2/connection.rb b/lib/async/http/protocol/http2/connection.rb index cfde0eb..46b6120 100644 --- a/lib/async/http/protocol/http2/connection.rb +++ b/lib/async/http/protocol/http2/connection.rb @@ -113,7 +113,7 @@ def read_in_background(parent: Task.current) attr :promises def peer - @peer ||= Protocol::HTTP::Peer.for(@stream.io) + @peer ||= ::Protocol::HTTP::Peer.for(@stream.io) end attr :count From 4ce90b1879ad80ade9ee2379ea86e85789ecd689 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 10 Nov 2024 01:19:25 +1300 Subject: [PATCH 105/125] Bump patch version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 6fe9ed5..9381714 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.83.0" + VERSION = "0.83.1" end end From 08878c4428fddb4b20b343ff643fa9ca8eb3c006 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 20 Nov 2024 22:26:20 +1300 Subject: [PATCH 106/125] Fix tests. --- fixtures/async/http/a_protocol.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fixtures/async/http/a_protocol.rb b/fixtures/async/http/a_protocol.rb index 82eb3de..c20a7de 100644 --- a/fixtures/async/http/a_protocol.rb +++ b/fixtures/async/http/a_protocol.rb @@ -229,6 +229,11 @@ module HTTP end end + def endpoint_options + # Add a timeout to ensure that slow clients are disconnected: + super.merge(timeout: 0.5) + end + it "should have valid scheme" do expect(server.scheme).to be == "http" end @@ -507,6 +512,10 @@ module HTTP end end + def endpoint_options + super.merge(timeout: 0.5) + end + it "can't get /" do expect do response = client.get("/") From a16c4c4b005fea3fc29860dd8ff0c83e56771330 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 24 Nov 2024 20:10:58 +1300 Subject: [PATCH 107/125] Pass through *arguments and **options for consistency. (#191) --- lib/async/http/internet/instance.rb | 4 ++-- releases.md | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/async/http/internet/instance.rb b/lib/async/http/internet/instance.rb index 92a60c0..bdbcfc8 100644 --- a/lib/async/http/internet/instance.rb +++ b/lib/async/http/internet/instance.rb @@ -17,8 +17,8 @@ def self.instance class << self ::Protocol::HTTP::Methods.each do |name, verb| - define_method(verb.downcase) do |url, headers = nil, body = nil, &block| - self.instance.call(verb, url, headers, body, &block) + define_method(verb.downcase) do |url, *arguments, **options, &block| + self.instance.call(verb, url, *arguments, **options, &block) end end end diff --git a/releases.md b/releases.md index cf5c1b8..14a6e18 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - Minor consistency fixes to `Async::HTTP::Internet` singleton methods. + ## v0.82.0 - `protocol-http1` introduces a line length limit for request line, response line, header lines and chunk length lines. From f81e23817c6b08a7027c494cb2d6fadce8946409 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 24 Nov 2024 20:12:52 +1300 Subject: [PATCH 108/125] Bump minor version. --- lib/async/http/version.rb | 2 +- readme.md | 4 ++++ releases.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 9381714..f13628b 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.83.1" + VERSION = "0.84.0" end end diff --git a/readme.md b/readme.md index c574500..1d1a7ec 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,10 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. +### v0.84.0 + + - Minor consistency fixes to `Async::HTTP::Internet` singleton methods. + ### v0.82.0 - `protocol-http1` introduces a line length limit for request line, response line, header lines and chunk length lines. diff --git a/releases.md b/releases.md index 14a6e18..63286fc 100644 --- a/releases.md +++ b/releases.md @@ -1,6 +1,6 @@ # Releases -## Unreleased +## v0.84.0 - Minor consistency fixes to `Async::HTTP::Internet` singleton methods. From 4f0256b2ddabe0852a2ab3fc638ddddf5ccc80d2 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 28 Nov 2024 23:23:53 +1300 Subject: [PATCH 109/125] Remove support for HTTP/2 priority. (#192) --- async-http.gemspec | 2 +- lib/async/http/protocol/http2/output.rb | 2 ++ lib/async/http/protocol/http2/request.rb | 8 ++++---- lib/async/http/protocol/http2/response.rb | 4 ++-- lib/async/http/protocol/http2/stream.rb | 4 +++- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index b711ffa..e498ba0 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -30,7 +30,7 @@ Gem::Specification.new do |spec| spec.add_dependency "io-stream", "~> 0.6" spec.add_dependency "protocol-http", "~> 0.43" spec.add_dependency "protocol-http1", ">= 0.28.1" - spec.add_dependency "protocol-http2", "~> 0.19" + spec.add_dependency "protocol-http2", "~> 0.21" spec.add_dependency "traces", "~> 0.10" spec.add_dependency "metrics", "~> 0.12" end diff --git a/lib/async/http/protocol/http2/output.rb b/lib/async/http/protocol/http2/output.rb index 32ec3d8..f3538bb 100644 --- a/lib/async/http/protocol/http2/output.rb +++ b/lib/async/http/protocol/http2/output.rb @@ -37,6 +37,8 @@ def window_updated(size) @guard.synchronize do @window_updated.signal end + + return true end def write(chunk) diff --git a/lib/async/http/protocol/http2/request.rb b/lib/async/http/protocol/http2/request.rb index 84fdd5a..de418d8 100644 --- a/lib/async/http/protocol/http2/request.rb +++ b/lib/async/http/protocol/http2/request.rb @@ -112,7 +112,7 @@ def hijack? def send_response(response) if response.nil? - return @stream.send_headers(nil, NO_RESPONSE, ::Protocol::HTTP2::END_STREAM) + return @stream.send_headers(NO_RESPONSE, ::Protocol::HTTP2::END_STREAM) end protocol_headers = [ @@ -129,14 +129,14 @@ def send_response(response) # This function informs the headers object that any subsequent headers are going to be trailer. Therefore, it must be called *before* sending the headers, to avoid any race conditions. trailer = response.headers.trailer! - @stream.send_headers(nil, headers) + @stream.send_headers(headers) @stream.send_body(body, trailer) else # Ensure the response body is closed if we are ending the stream: response.close - @stream.send_headers(nil, headers, ::Protocol::HTTP2::END_STREAM) + @stream.send_headers(headers, ::Protocol::HTTP2::END_STREAM) end end @@ -149,7 +149,7 @@ def write_interim_response(status, headers = nil) interim_response_headers = ::Protocol::HTTP::Headers::Merged.new(interim_response_headers, headers) end - @stream.send_headers(nil, interim_response_headers) + @stream.send_headers(interim_response_headers) end end end diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index c952c47..5792f05 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -222,7 +222,7 @@ def send_request(request) ) if request.body.nil? - @stream.send_headers(nil, headers, ::Protocol::HTTP2::END_STREAM) + @stream.send_headers(headers, ::Protocol::HTTP2::END_STREAM) else if length = request.body.length # This puts it at the end of the pseudo-headers: @@ -233,7 +233,7 @@ def send_request(request) trailer = request.headers.trailer! begin - @stream.send_headers(nil, headers) + @stream.send_headers(headers) rescue raise RequestFailed end diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index cf50fdb..3c077d2 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -131,7 +131,7 @@ def finish_output(error = nil) else # Write trailer? if trailer&.any? - send_headers(nil, trailer, ::Protocol::HTTP2::END_STREAM) + send_headers(trailer, ::Protocol::HTTP2::END_STREAM) else send_data(nil, ::Protocol::HTTP2::END_STREAM) end @@ -142,6 +142,8 @@ def window_updated(size) super @output&.window_updated(size) + + return true end # When the stream transitions to the closed state, this method is called. There are roughly two ways this can happen: From a3e7a0e06c5a6219dbdfd6c9145501464f71d693 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 28 Nov 2024 23:31:23 +1300 Subject: [PATCH 110/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index f13628b..7c053ff 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.84.0" + VERSION = "0.85.0" end end From 8ce026e1c541d1cdf870bfc98b17e12861c0ce65 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 1 Dec 2024 21:04:25 +1300 Subject: [PATCH 111/125] Add support for `Protocol::HTTP2::Settings::NO_RFC7540_PRIORITIES`. --- async-http.gemspec | 2 +- lib/async/http/protocol/http2.rb | 2 ++ releases.md | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/async-http.gemspec b/async-http.gemspec index e498ba0..5aeb9fd 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -30,7 +30,7 @@ Gem::Specification.new do |spec| spec.add_dependency "io-stream", "~> 0.6" spec.add_dependency "protocol-http", "~> 0.43" spec.add_dependency "protocol-http1", ">= 0.28.1" - spec.add_dependency "protocol-http2", "~> 0.21" + spec.add_dependency "protocol-http2", "~> 0.22" spec.add_dependency "traces", "~> 0.10" spec.add_dependency "metrics", "~> 0.12" end diff --git a/lib/async/http/protocol/http2.rb b/lib/async/http/protocol/http2.rb index 3222f00..e7a5420 100644 --- a/lib/async/http/protocol/http2.rb +++ b/lib/async/http/protocol/http2.rb @@ -27,6 +27,7 @@ def self.trailer? ::Protocol::HTTP2::Settings::ENABLE_PUSH => 0, ::Protocol::HTTP2::Settings::MAXIMUM_FRAME_SIZE => 0x100000, ::Protocol::HTTP2::Settings::INITIAL_WINDOW_SIZE => 0x800000, + ::Protocol::HTTP2::Settings::NO_RFC7540_PRIORITIES => 1, } SERVER_SETTINGS = { @@ -35,6 +36,7 @@ def self.trailer? ::Protocol::HTTP2::Settings::MAXIMUM_FRAME_SIZE => 0x100000, ::Protocol::HTTP2::Settings::INITIAL_WINDOW_SIZE => 0x800000, ::Protocol::HTTP2::Settings::ENABLE_CONNECT_PROTOCOL => 1, + ::Protocol::HTTP2::Settings::NO_RFC7540_PRIORITIES => 1, } def self.client(peer, settings = CLIENT_SETTINGS) diff --git a/releases.md b/releases.md index 63286fc..54346ff 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - Add support for HTTP/2 `NO_RFC7540_PRIORITIES`. See for more details. + ## v0.84.0 - Minor consistency fixes to `Async::HTTP::Internet` singleton methods. From e2598452c5d014623cee16b451f1cb84e4e74f27 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 1 Dec 2024 23:51:40 +1300 Subject: [PATCH 112/125] Bump minor version. --- lib/async/http/version.rb | 2 +- readme.md | 4 ++++ releases.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 7c053ff..0593eea 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.85.0" + VERSION = "0.86.0" end end diff --git a/readme.md b/readme.md index 1d1a7ec..04e24a4 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,10 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. +### v0.86.0 + + - Add support for HTTP/2 `NO_RFC7540_PRIORITIES`. See for more details. + ### v0.84.0 - Minor consistency fixes to `Async::HTTP::Internet` singleton methods. diff --git a/releases.md b/releases.md index 54346ff..9de209f 100644 --- a/releases.md +++ b/releases.md @@ -1,6 +1,6 @@ # Releases -## Unreleased +## v0.86.0 - Add support for HTTP/2 `NO_RFC7540_PRIORITIES`. See for more details. From 36b86c6f542fcf82d01c3fac5c6ddf5f04e07c24 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jan 2025 01:09:06 +1300 Subject: [PATCH 113/125] Avoid temporary array allocation. --- async-http.gemspec | 2 +- lib/async/http/protocol/http1/request.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index 5aeb9fd..1104220 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -29,7 +29,7 @@ Gem::Specification.new do |spec| spec.add_dependency "io-endpoint", "~> 0.14" spec.add_dependency "io-stream", "~> 0.6" spec.add_dependency "protocol-http", "~> 0.43" - spec.add_dependency "protocol-http1", ">= 0.28.1" + spec.add_dependency "protocol-http1", "~> 0.29" spec.add_dependency "protocol-http2", "~> 0.22" spec.add_dependency "traces", "~> 0.10" spec.add_dependency "metrics", "~> 0.12" diff --git a/lib/async/http/protocol/http1/request.rb b/lib/async/http/protocol/http1/request.rb index 9408be2..21edf4d 100644 --- a/lib/async/http/protocol/http1/request.rb +++ b/lib/async/http/protocol/http1/request.rb @@ -11,8 +11,8 @@ module Protocol module HTTP1 class Request < Protocol::Request def self.read(connection) - if parts = connection.read_request - self.new(connection, *parts) + connection.read_request do |authority, method, target, version, headers, body| + self.new(connection, authority, method, target, version, headers, body) end end From 714ed4a556487311461fcdeebc7e1c58b8d573a5 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jan 2025 18:52:47 +1300 Subject: [PATCH 114/125] Unify `CONNECT` semantics. --- async-http.gemspec | 4 +-- lib/async/http/protocol/http1/client.rb | 11 ++++++++- lib/async/http/protocol/http1/request.rb | 30 ++++++++++++++++++++--- lib/async/http/protocol/http2/request.rb | 2 +- lib/async/http/protocol/http2/response.rb | 5 +++- lib/async/http/proxy.rb | 2 +- releases.md | 26 ++++++++++++++++++++ test/async/http/proxy.rb | 6 ++--- 8 files changed, 74 insertions(+), 12 deletions(-) diff --git a/async-http.gemspec b/async-http.gemspec index 1104220..fc08714 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,8 +28,8 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.9" spec.add_dependency "io-endpoint", "~> 0.14" spec.add_dependency "io-stream", "~> 0.6" - spec.add_dependency "protocol-http", "~> 0.43" - spec.add_dependency "protocol-http1", "~> 0.29" + spec.add_dependency "protocol-http", "~> 0.49" + spec.add_dependency "protocol-http1", "~> 0.30" spec.add_dependency "protocol-http2", "~> 0.22" spec.add_dependency "traces", "~> 0.10" spec.add_dependency "metrics", "~> 0.12" diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index fd478e5..1b32bd1 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -39,7 +39,16 @@ def call(request, task: Task.current) # We carefully interpret https://tools.ietf.org/html/rfc7230#section-6.3.1 to implement this correctly. begin - write_request(request.authority, request.method, request.path, @version, request.headers) + target = request.path + authority = request.authority + + # If we are using a CONNECT request, we need to use the authority as the target: + if request.connect? + target = authority + authority = nil + end + + write_request(authority, request.method, target, @version, request.headers) rescue # If we fail to fully write the request and body, we can retry this request. raise RequestFailed diff --git a/lib/async/http/protocol/http1/request.rb b/lib/async/http/protocol/http1/request.rb index 21edf4d..ad02fbc 100644 --- a/lib/async/http/protocol/http1/request.rb +++ b/lib/async/http/protocol/http1/request.rb @@ -10,21 +10,45 @@ module HTTP module Protocol module HTTP1 class Request < Protocol::Request + def self.valid_path?(target) + if target.start_with?("/") + return true + elsif target == '*' + return true + else + return false + end + end + + URI_PATTERN = %r{\A(?[^:/]+)://(?[^/]+)(?.*)\z} + def self.read(connection) connection.read_request do |authority, method, target, version, headers, body| - self.new(connection, authority, method, target, version, headers, body) + if method == ::Protocol::HTTP::Methods::CONNECT + # We put the target into the authority field for CONNECT requests, as per HTTP/2 semantics. + self.new(connection, nil, target, method, nil, version, headers, body) + elsif valid_path?(target) + # This is a valid request. + self.new(connection, nil, authority, method, target, version, headers, body) + elsif match = target.match(URI_PATTERN) + # We map the incoming absolute URI target to the scheme, authority, and path fields of the request. + self.new(connection, match[:scheme], match[:authority], method, match[:path], version, headers, body) + else + # This is an invalid request. + raise ::Protocol::HTTP1::BadRequest.new("Invalid request target: #{target}") + end end end UPGRADE = "upgrade" - def initialize(connection, authority, method, path, version, headers, body) + def initialize(connection, scheme, authority, method, path, version, headers, body) @connection = connection # HTTP/1 requests with an upgrade header (which can contain zero or more values) are extracted into the protocol field of the request, and we expect a response to select one of those protocols with a status code of 101 Switching Protocols. protocol = headers.delete("upgrade") - super(nil, authority, method, path, version, headers, body, protocol, self.public_method(:write_interim_response)) + super(scheme, authority, method, path, version, headers, body, protocol, self.public_method(:write_interim_response)) end def connection diff --git a/lib/async/http/protocol/http2/request.rb b/lib/async/http/protocol/http2/request.rb index de418d8..c49312a 100644 --- a/lib/async/http/protocol/http2/request.rb +++ b/lib/async/http/protocol/http2/request.rb @@ -99,7 +99,7 @@ def connection end def valid? - @scheme and @method and @path + @scheme and @method and (@path or @method == ::Protocol::HTTP::Methods::CONNECT) end def hijack? diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index 5792f05..d127ab1 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -204,9 +204,12 @@ def send_request(request) pseudo_headers = [ [SCHEME, request.scheme], [METHOD, request.method], - [PATH, request.path], ] + if path = request.path + pseudo_headers << [PATH, path] + end + # To ensure that the HTTP/1.1 request line can be reproduced accurately, this pseudo-header field MUST be omitted when translating from an HTTP/1.1 request that has a request target in origin or asterisk form (see [RFC7230], Section 5.3). Clients that generate HTTP/2 requests directly SHOULD use the :authority pseudo-header field instead of the Host header field. if authority = request.authority pseudo_headers << [AUTHORITY, authority] diff --git a/lib/async/http/proxy.rb b/lib/async/http/proxy.rb index b6fd204..d6e86e8 100644 --- a/lib/async/http/proxy.rb +++ b/lib/async/http/proxy.rb @@ -82,7 +82,7 @@ def close def connect(&block) input = Body::Writable.new - response = @client.connect(@address.to_s, @headers, input) + response = @client.connect(authority: @address, headers: @headers, body: input) if response.success? pipe = Body::Pipe.new(response.body, input) diff --git a/releases.md b/releases.md index 9de209f..9ae667c 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,31 @@ # Releases +## Unreleased + +### Unify HTTP/1 and HTTP/2 `CONNECT` semantics + +HTTP/1 has a request line "target" which takes different forms depending on the kind of request. For `CONNECT` requests, the target is the authority (host and port) of the target server, e.g. + +``` +CONNECT example.com:443 HTTP/1.1 +``` + +In HTTP/2, the `CONNECT` method uses the `:authority` pseudo-header to specify the target, e.g. + +```http +[HEADERS FRAME] +:method: connect +:authority: example.com:443 +``` + +In HTTP/1, the `Request#path` attribute was previously used to store the target, and this was incorrectly mapped to the `:path` pseudo-header in HTTP/2. This has been corrected, and the `Request#authority` attribute is now used to store the target for both HTTP/1 and HTTP/2, and mapped accordingly. Thus, to make a `CONNECT` request, you should set the `Request#authority` attribute, e.g. + +```ruby +response = client.connect(authority: "example.com:443") +``` + +For HTTP/1, the authority is mapped back to the request line target, and for HTTP/2, it is mapped to the `:authority` pseudo-header. + ## v0.86.0 - Add support for HTTP/2 `NO_RFC7540_PRIORITIES`. See for more details. diff --git a/test/async/http/proxy.rb b/test/async/http/proxy.rb index fe8d180..1a1b280 100644 --- a/test/async/http/proxy.rb +++ b/test/async/http/proxy.rb @@ -54,7 +54,7 @@ it "can connect and hijack connection" do input = Async::HTTP::Body::Writable.new - response = client.connect("127.0.0.1:1234", [], input) + response = client.connect(body: input, authority: "127.0.0.1:1234") expect(response).to be(:success?) @@ -68,7 +68,7 @@ with "echo server" do let(:app) do Protocol::HTTP::Middleware.for do |request| - expect(request.path).to be == "localhost:1" + expect(request.authority).to be == "localhost:1" Async::HTTP::Body::Hijack.response(request, 200, {}) do |stream| while chunk = stream.read_partial(1024) @@ -125,7 +125,7 @@ next Protocol::HTTP::Response[407, [], nil] end - host, port = request.path.split(":", 2) + host, port = request.authority.split(":", 2) endpoint = IO::Endpoint.tcp(host, port) Console.logger.debug(self) {"Making connection to #{endpoint}..."} From f9e59862ac5652edfd3c086e0f42d0f5426009ae Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jan 2025 18:54:13 +1300 Subject: [PATCH 115/125] Modernize code. --- .github/workflows/documentation-coverage.yaml | 2 +- .github/workflows/documentation.yaml | 2 +- .github/workflows/test-coverage.yaml | 4 ++-- .github/workflows/test-external.yaml | 1 + .github/workflows/test.yaml | 1 + async-http.gemspec | 2 +- lib/async/http/protocol/http1/client.rb | 2 +- lib/async/http/protocol/http1/request.rb | 4 ++-- lib/async/http/protocol/http2/request.rb | 2 +- lib/async/http/protocol/http2/response.rb | 2 +- lib/async/http/proxy.rb | 2 +- license.md | 2 +- test/async/http/proxy.rb | 2 +- 13 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/documentation-coverage.yaml b/.github/workflows/documentation-coverage.yaml index b3bac9a..8d801c5 100644 --- a/.github/workflows/documentation-coverage.yaml +++ b/.github/workflows/documentation-coverage.yaml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.3" + ruby-version: "3.4" bundler-cache: true - name: Validate coverage diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index f5f553a..e47c6b3 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -29,7 +29,7 @@ jobs: - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.3" + ruby-version: "3.4" bundler-cache: true - name: Installing packages diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 50e9293..e6dc5c3 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -21,7 +21,7 @@ jobs: - macos ruby: - - "3.3" + - "3.4" steps: - uses: actions/checkout@v4 @@ -49,7 +49,7 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.3" + ruby-version: "3.4" bundler-cache: true - uses: actions/download-artifact@v4 diff --git a/.github/workflows/test-external.yaml b/.github/workflows/test-external.yaml index 21898f5..c9cc200 100644 --- a/.github/workflows/test-external.yaml +++ b/.github/workflows/test-external.yaml @@ -23,6 +23,7 @@ jobs: - "3.1" - "3.2" - "3.3" + - "3.4" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b9f70e9..4b87012 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -26,6 +26,7 @@ jobs: - "3.1" - "3.2" - "3.3" + - "3.4" experimental: [false] diff --git a/async-http.gemspec b/async-http.gemspec index fc08714..6565546 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -28,9 +28,9 @@ Gem::Specification.new do |spec| spec.add_dependency "async-pool", "~> 0.9" spec.add_dependency "io-endpoint", "~> 0.14" spec.add_dependency "io-stream", "~> 0.6" + spec.add_dependency "metrics", "~> 0.12" spec.add_dependency "protocol-http", "~> 0.49" spec.add_dependency "protocol-http1", "~> 0.30" spec.add_dependency "protocol-http2", "~> 0.22" spec.add_dependency "traces", "~> 0.10" - spec.add_dependency "metrics", "~> 0.12" end diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index 1b32bd1..0a4ee2c 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. require_relative "connection" diff --git a/lib/async/http/protocol/http1/request.rb b/lib/async/http/protocol/http1/request.rb index ad02fbc..5e12b00 100644 --- a/lib/async/http/protocol/http1/request.rb +++ b/lib/async/http/protocol/http1/request.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. require_relative "../request" @@ -13,7 +13,7 @@ class Request < Protocol::Request def self.valid_path?(target) if target.start_with?("/") return true - elsif target == '*' + elsif target == "*" return true else return false diff --git a/lib/async/http/protocol/http2/request.rb b/lib/async/http/protocol/http2/request.rb index c49312a..004784a 100644 --- a/lib/async/http/protocol/http2/request.rb +++ b/lib/async/http/protocol/http2/request.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. require_relative "../request" require_relative "stream" diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index d127ab1..79bdc54 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. require_relative "../response" require_relative "stream" diff --git a/lib/async/http/proxy.rb b/lib/async/http/proxy.rb index d6e86e8..4d68dd8 100644 --- a/lib/async/http/proxy.rb +++ b/lib/async/http/proxy.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2019-2024, by Samuel Williams. +# Copyright, 2019-2025, by Samuel Williams. require_relative "client" require_relative "endpoint" diff --git a/license.md b/license.md index daa72d1..992c353 100644 --- a/license.md +++ b/license.md @@ -1,6 +1,6 @@ # MIT License -Copyright, 2017-2024, by Samuel Williams. +Copyright, 2017-2025, by Samuel Williams. Copyright, 2018, by Viacheslav Koval. Copyright, 2018, by Janko Marohnić. Copyright, 2019, by Denis Talakevich. diff --git a/test/async/http/proxy.rb b/test/async/http/proxy.rb index 1a1b280..b82c905 100644 --- a/test/async/http/proxy.rb +++ b/test/async/http/proxy.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2019-2024, by Samuel Williams. +# Copyright, 2019-2025, by Samuel Williams. # Copyright, 2020, by Sam Shadwell. require "async" From 5bded8ae9eb16b2d9547dcd330c78e81f9bedd35 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jan 2025 19:11:35 +1300 Subject: [PATCH 116/125] Bump minor version. --- lib/async/http/version.rb | 2 +- readme.md | 4 ++++ releases.md | 10 ++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 0593eea..9bfe6ba 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.86.0" + VERSION = "0.87.0" end end diff --git a/readme.md b/readme.md index 04e24a4..f15c6f4 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,10 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. +### v0.87.0 + + - [Unify HTTP/1 and HTTP/2 `CONNECT` semantics](https://socketry.github.io/async-http/releases/index#unify-http/1-and-http/2-connect-semantics) + ### v0.86.0 - Add support for HTTP/2 `NO_RFC7540_PRIORITIES`. See for more details. diff --git a/releases.md b/releases.md index 9ae667c..59bd24c 100644 --- a/releases.md +++ b/releases.md @@ -1,18 +1,16 @@ # Releases -## Unreleased +## v0.87.0 ### Unify HTTP/1 and HTTP/2 `CONNECT` semantics HTTP/1 has a request line "target" which takes different forms depending on the kind of request. For `CONNECT` requests, the target is the authority (host and port) of the target server, e.g. -``` -CONNECT example.com:443 HTTP/1.1 -``` + CONNECT example.com:443 HTTP/1.1 In HTTP/2, the `CONNECT` method uses the `:authority` pseudo-header to specify the target, e.g. -```http +``` http [HEADERS FRAME] :method: connect :authority: example.com:443 @@ -20,7 +18,7 @@ In HTTP/2, the `CONNECT` method uses the `:authority` pseudo-header to specify t In HTTP/1, the `Request#path` attribute was previously used to store the target, and this was incorrectly mapped to the `:path` pseudo-header in HTTP/2. This has been corrected, and the `Request#authority` attribute is now used to store the target for both HTTP/1 and HTTP/2, and mapped accordingly. Thus, to make a `CONNECT` request, you should set the `Request#authority` attribute, e.g. -```ruby +``` ruby response = client.connect(authority: "example.com:443") ``` From 02464cf45116fa0ab6a79f25c2efbdb8376b00cb Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 8 Feb 2025 12:05:22 +1300 Subject: [PATCH 117/125] Remove count, it's not particularly useful. --- lib/async/http/protocol/http2/client.rb | 2 -- lib/async/http/protocol/http2/connection.rb | 3 +-- lib/async/http/protocol/http2/server.rb | 2 -- test/async/http/protocol/http2.rb | 2 +- 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/async/http/protocol/http2/client.rb b/lib/async/http/protocol/http2/client.rb index 5ad422f..f59b4b6 100644 --- a/lib/async/http/protocol/http2/client.rb +++ b/lib/async/http/protocol/http2/client.rb @@ -32,8 +32,6 @@ def create_response def call(request) raise ::Protocol::HTTP2::Error, "Connection closed!" if self.closed? - @count += 1 - response = create_response write_request(response, request) read_response(response) diff --git a/lib/async/http/protocol/http2/connection.rb b/lib/async/http/protocol/http2/connection.rb index 46b6120..7757703 100644 --- a/lib/async/http/protocol/http2/connection.rb +++ b/lib/async/http/protocol/http2/connection.rb @@ -29,7 +29,6 @@ module Connection def initialize(*) super - @count = 0 @reader = nil # Writing multiple frames at the same time can cause odd problems if frames are only partially written. So we use a semaphore to ensure frames are written in their entirety. @@ -41,7 +40,7 @@ def synchronize(&block) end def to_s - "\#<#{self.class} #{@count} requests, #{@streams.count} active streams>" + "\#<#{self.class} #{@streams.count} active streams>" end def as_json(...) diff --git a/lib/async/http/protocol/http2/server.rb b/lib/async/http/protocol/http2/server.rb index 0372955..32e2d6e 100644 --- a/lib/async/http/protocol/http2/server.rb +++ b/lib/async/http/protocol/http2/server.rb @@ -51,8 +51,6 @@ def each(task: Task.current) @requests&.async do |task, request| task.annotate("Incoming request: #{request.method} #{request.path.inspect}.") - @count += 1 - task.defer_stop do response = yield(request) rescue diff --git a/test/async/http/protocol/http2.rb b/test/async/http/protocol/http2.rb index bfef591..cd7d308 100644 --- a/test/async/http/protocol/http2.rb +++ b/test/async/http/protocol/http2.rb @@ -17,7 +17,7 @@ response = client.get("/") connection = response.connection - expect(connection.as_json).to be =~ /#/ + expect(connection.as_json).to be =~ /#/ ensure response&.close end From 9ae1f17751cd90abb76fd8a4f1bf6277af40ce82 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 8 Feb 2025 12:07:00 +1300 Subject: [PATCH 118/125] Modernize logging. --- bake/async/http/h2spec.rb | 2 +- examples/download/chunked.rb | 12 ++++++------ examples/google/search.rb | 6 +++--- examples/licenses/list.rb | 2 +- examples/stream/stop.rb | 6 +++--- lib/async/http/client.rb | 4 ++-- lib/async/http/protocol/http1/client.rb | 2 +- lib/async/http/protocol/http2/stream.rb | 2 +- lib/async/http/protocol/https.rb | 2 +- lib/async/http/server.rb | 4 ++-- test/async/http/proxy.rb | 8 ++++---- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/bake/async/http/h2spec.rb b/bake/async/http/h2spec.rb index 960e8b3..9aec9e3 100644 --- a/bake/async/http/h2spec.rb +++ b/bake/async/http/h2spec.rb @@ -30,7 +30,7 @@ def server container = Async::Container.new - Console.logger.info(self){"Starting server..."} + Console.info(self){"Starting server..."} container.run(count: 1) do server = Async::HTTP::Server.for(endpoint, protocol: Async::HTTP::Protocol::HTTP2, scheme: "https") do |request| diff --git a/examples/download/chunked.rb b/examples/download/chunked.rb index f2b3410..1a40943 100755 --- a/examples/download/chunked.rb +++ b/examples/download/chunked.rb @@ -20,7 +20,7 @@ headers = {"user-agent" => "curl/7.69.1", "accept" => "*/*"} file = File.open("products.csv", "w") - Console.logger.info(self) {"Saving download to #{Dir.pwd}"} + Console.info(self) {"Saving download to #{Dir.pwd}"} begin response = client.head(endpoint.path, headers) @@ -39,7 +39,7 @@ response&.close end - Console.logger.info(self) {"Content length: #{content_length/(1024**2)}MiB"} + Console.info(self) {"Content length: #{content_length/(1024**2)}MiB"} parts = [] offset = 0 @@ -56,7 +56,7 @@ offset += chunk_size end - Console.logger.info(self) {"Breaking download into #{parts.size} parts..."} + Console.info(self) {"Breaking download into #{parts.size} parts..."} semaphore = Async::Semaphore.new(8) barrier = Async::Barrier.new(parent: semaphore) @@ -65,7 +65,7 @@ barrier.async do part = parts.shift - Console.logger.info(self) {"Issuing range request range: bytes=#{part.min}-#{part.max}"} + Console.info(self) {"Issuing range request range: bytes=#{part.min}-#{part.max}"} response = client.get(endpoint.path, [ ["range", "bytes=#{part.min}-#{part.max-1}"], @@ -73,13 +73,13 @@ ]) if response.success? - Console.logger.info(self) {"Got response: #{response}... writing data for #{part}."} + Console.info(self) {"Got response: #{response}... writing data for #{part}."} written = file.pwrite(response.read, part.min) amount += written duration = Async::Clock.now - start_time - Console.logger.info(self) {"Rate: #{((amount.to_f/(1024**2))/duration).round(2)}MiB/s"} + Console.info(self) {"Rate: #{((amount.to_f/(1024**2))/duration).round(2)}MiB/s"} end end end diff --git a/examples/google/search.rb b/examples/google/search.rb index d976584..bbe1ec6 100755 --- a/examples/google/search.rb +++ b/examples/google/search.rb @@ -14,7 +14,7 @@ class Google < Protocol::HTTP::Middleware def search(term) - Console.logger.info(self) {"Searching for #{term}..."} + Console.info(self) {"Searching for #{term}..."} self.get("/search?q=#{term}", {"user-agent" => "Hi Google!"}) end @@ -40,10 +40,10 @@ def search(term) end end.map(&:wait).to_h - Console.logger.info(self, name: "counts") {counts} + Console.info(self, name: "counts") {counts} end - Console.logger.info(self, name: "duration") {duration} + Console.info(self, name: "duration") {duration} ensure google.close end diff --git a/examples/licenses/list.rb b/examples/licenses/list.rb index dd892e9..3e03524 100755 --- a/examples/licenses/list.rb +++ b/examples/licenses/list.rb @@ -63,7 +63,7 @@ def fetch_rubygem_license(name, version) rescue RateLimitingError response.finish - Console.logger.warn(name) {"Rate limited..."} + Console.warn(name) {"Rate limited..."} Async::Task.current.sleep(1.0) retry diff --git a/examples/stream/stop.rb b/examples/stream/stop.rb index c31f5a1..a4e1cc1 100644 --- a/examples/stream/stop.rb +++ b/examples/stream/stop.rb @@ -16,16 +16,16 @@ connection = response.connection response.each do |chunk| - Console.logger.info(response) {chunk} + Console.info(response) {chunk} end ensure - Console.logger.info(response) {"Closing response..."} + Console.info(response) {"Closing response..."} response&.close end parent.sleep(5) - Console.logger.info(parent) {"Killing #{child}..."} + Console.info(parent) {"Killing #{child}..."} child.stop ensure internet&.close diff --git a/lib/async/http/client.rb b/lib/async/http/client.rb index 73c71de..743cba0 100755 --- a/lib/async/http/client.rb +++ b/lib/async/http/client.rb @@ -81,7 +81,7 @@ def self.open(*arguments, **options, &block) def close while @pool.busy? - Console.logger.warn(self) {"Waiting for #{@protocol} pool to drain: #{@pool}"} + Console.warn(self) {"Waiting for #{@protocol} pool to drain: #{@pool}"} @pool.wait end @@ -164,7 +164,7 @@ def make_pool(**options) self.assign_default_tags(options[:tags] ||= {}) Async::Pool::Controller.wrap(**options) do - Console.logger.debug(self) {"Making connection to #{@endpoint.inspect}"} + Console.debug(self) {"Making connection to #{@endpoint.inspect}"} @protocol.client(@endpoint.connect) end diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index 0a4ee2c..30cf6e5 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -32,7 +32,7 @@ def closed(error = nil) # Used by the client to send requests to the remote server. def call(request, task: Task.current) - Console.logger.debug(self) {"#{request.method} #{request.path} #{request.headers.inspect}"} + Console.debug(self) {"#{request.method} #{request.path} #{request.headers.inspect}"} # Mark the start of the trailers: trailer = request.headers.trailer! diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index 3c077d2..ec947ae 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -65,7 +65,7 @@ def process_headers(frame) @input.close_write end rescue ::Protocol::HTTP2::HeaderError => error - Console.logger.debug(self, error) + Console.debug(self, "Error while processing headers!", error: error) send_reset_stream(error.code) end diff --git a/lib/async/http/protocol/https.rb b/lib/async/http/protocol/https.rb index fd5b07e..a79784a 100644 --- a/lib/async/http/protocol/https.rb +++ b/lib/async/http/protocol/https.rb @@ -25,7 +25,7 @@ def self.protocol_for(peer) # alpn_protocol is only available if openssl v1.0.2+ name = peer.alpn_protocol - Console.logger.debug(self) {"Negotiating protocol #{name.inspect}..."} + Console.debug(self) {"Negotiating protocol #{name.inspect}..."} if protocol = HANDLERS[name] return protocol diff --git a/lib/async/http/server.rb b/lib/async/http/server.rb index be0a536..3fa3ade 100755 --- a/lib/async/http/server.rb +++ b/lib/async/http/server.rb @@ -45,14 +45,14 @@ def to_json(...) def accept(peer, address, task: Task.current) connection = @protocol.server(peer) - Console.logger.debug(self) {"Incoming connnection from #{address.inspect} to #{@protocol}"} + Console.debug(self) {"Incoming connnection from #{address.inspect} to #{@protocol}"} connection.each do |request| # We set the default scheme unless it was otherwise specified. # https://tools.ietf.org/html/rfc7230#section-5.5 request.scheme ||= self.scheme - # Console.logger.debug(self) {"Incoming request from #{address.inspect}: #{request.method} #{request.path}"} + # Console.debug(self) {"Incoming request from #{address.inspect}: #{request.method} #{request.path}"} # If this returns nil, we assume that the connection has been hijacked. self.call(request) diff --git a/test/async/http/proxy.rb b/test/async/http/proxy.rb index b82c905..8aeff64 100644 --- a/test/async/http/proxy.rb +++ b/test/async/http/proxy.rb @@ -128,11 +128,11 @@ host, port = request.authority.split(":", 2) endpoint = IO::Endpoint.tcp(host, port) - Console.logger.debug(self) {"Making connection to #{endpoint}..."} + Console.debug(self) {"Making connection to #{endpoint}..."} Async::HTTP::Body::Hijack.response(request, 200, {}) do |stream| upstream = ::IO::Stream::Buffered.wrap(endpoint.connect) - Console.logger.debug(self) {"Connected to #{upstream}..."} + Console.debug(self) {"Connected to #{upstream}..."} reader = Async do |task| task.annotate "Upstream reader." @@ -142,7 +142,7 @@ stream.flush end ensure - Console.logger.debug(self) {"Finished reading from upstream..."} + Console.debug(self) {"Finished reading from upstream..."} stream.close_write end @@ -154,7 +154,7 @@ upstream.flush end ensure - Console.logger.debug(self) {"Finished writing to upstream..."} + Console.debug(self) {"Finished writing to upstream..."} upstream.close_write end From 74441ce1345eb573078bbea44e54bebf74e4810c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 30 Jan 2025 15:48:13 +1300 Subject: [PATCH 119/125] Tidy up logging. --- lib/async/http/protocol/http1/client.rb | 2 -- lib/async/http/protocol/http1/server.rb | 2 +- lib/async/http/protocol/http2/stream.rb | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/async/http/protocol/http1/client.rb b/lib/async/http/protocol/http1/client.rb index 30cf6e5..a612468 100644 --- a/lib/async/http/protocol/http1/client.rb +++ b/lib/async/http/protocol/http1/client.rb @@ -32,8 +32,6 @@ def closed(error = nil) # Used by the client to send requests to the remote server. def call(request, task: Task.current) - Console.debug(self) {"#{request.method} #{request.path} #{request.headers.inspect}"} - # Mark the start of the trailers: trailer = request.headers.trailer! diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index bec8dd0..2ebf1ca 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -34,7 +34,7 @@ def fail_request(status) write_body(@version, nil) rescue => error # At this point, there is very little we can do to recover: - Console::Event::Failure.for(error).emit(self, "Failed to write failure response!", severity: :debug) + Console.debug(self, "Failed to write failure response!", error) end def next_request diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index ec947ae..0ed5d5f 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -65,7 +65,7 @@ def process_headers(frame) @input.close_write end rescue ::Protocol::HTTP2::HeaderError => error - Console.debug(self, "Error while processing headers!", error: error) + Console.debug(self, "Error while processing headers!", error) send_reset_stream(error.code) end From 4308c8289b1ce552cf198a272bbb5305a1bf5a8c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 13 Mar 2025 20:26:29 +1300 Subject: [PATCH 120/125] Fix documentation reference. --- lib/async/http/endpoint.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index e8579bf..4a67f72 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -40,7 +40,7 @@ def self.parse(string, endpoint = nil, **options) # # @parameter scheme [String] The scheme to use, e.g. "http" or "https". # @parameter hostname [String] The hostname to connect to (or bind to). - # @parameter *options [Hash] Additional options, passed to {#initialize}. + # @parameter *options [Hash] Additional options, passed to {initialize}. def self.for(scheme, hostname, path = "/", **options) # TODO: Consider using URI.for once it becomes available: uri_klass = SCHEMES.fetch(scheme.downcase) do From c0618bacc3e51b4d4530603815bc9df22bfc40a6 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 13 Mar 2025 21:39:53 +1300 Subject: [PATCH 121/125] Allow passing through configuration to underlying protocol. (#198) --- lib/async/http/protocol/configurable.rb | 43 +++++++++++++++ lib/async/http/protocol/defaulton.rb | 36 ++++++++++++ lib/async/http/protocol/http.rb | 54 ++++++++++++------ lib/async/http/protocol/http1.rb | 23 ++++++-- lib/async/http/protocol/http1/connection.rb | 5 +- lib/async/http/protocol/http10.rb | 21 +++++-- lib/async/http/protocol/http11.rb | 21 +++++-- lib/async/http/protocol/http2.rb | 21 ++++++- lib/async/http/protocol/http2/connection.rb | 2 +- lib/async/http/protocol/https.rb | 54 ++++++++++++++---- releases.md | 61 +++++++++++++++++++++ test/async/http/protocol/http.rb | 32 +++++++++++ test/async/http/protocol/http1.rb | 24 ++++++++ test/async/http/protocol/https.rb | 54 ++++++++++++++++++ 14 files changed, 408 insertions(+), 43 deletions(-) create mode 100644 lib/async/http/protocol/configurable.rb create mode 100644 lib/async/http/protocol/defaulton.rb create mode 100644 test/async/http/protocol/http1.rb create mode 100644 test/async/http/protocol/https.rb diff --git a/lib/async/http/protocol/configurable.rb b/lib/async/http/protocol/configurable.rb new file mode 100644 index 0000000..b8c7178 --- /dev/null +++ b/lib/async/http/protocol/configurable.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2025, by Samuel Williams. + +module Async + module HTTP + module Protocol + class Configured + def initialize(protocol, **options) + @protocol = protocol + @options = options + end + + # @attribute [Protocol] The underlying protocol. + attr :protocol + + # @attribute [Hash] The options to pass to the protocol. + attr :options + + def client(peer, **options) + options = @options.merge(options) + @protocol.client(peer, **options) + end + + def server(peer, **options) + options = @options.merge(options) + @protocol.server(peer, **options) + end + + def names + @protocol.names + end + end + + module Configurable + def new(**options) + Configured.new(self, **options) + end + end + end + end +end diff --git a/lib/async/http/protocol/defaulton.rb b/lib/async/http/protocol/defaulton.rb new file mode 100644 index 0000000..d9a915b --- /dev/null +++ b/lib/async/http/protocol/defaulton.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2025, by Samuel Williams. + +module Async + module HTTP + module Protocol + # This module provides a default instance of the protocol, which can be used to create clients and servers. The name is a play on "Default" + "Singleton". + module Defaulton + def self.extended(base) + base.instance_variable_set(:@default, base.new) + end + + attr_accessor :default + + # Create a client for an outbound connection, using the default instance. + def client(peer, **options) + default.client(peer, **options) + end + + # Create a server for an inbound connection, using the default instance. + def server(peer, **options) + default.server(peer, **options) + end + + # @returns [Array] The names of the supported protocol, used for Application Layer Protocol Negotiation (ALPN), using the default instance. + def names + default.names + end + end + + private_constant :Defaulton + end + end +end diff --git a/lib/async/http/protocol/http.rb b/lib/async/http/protocol/http.rb index b70bb83..3864222 100644 --- a/lib/async/http/protocol/http.rb +++ b/lib/async/http/protocol/http.rb @@ -4,19 +4,33 @@ # Copyright, 2024, by Thomas Morgan. # Copyright, 2024, by Samuel Williams. +require_relative "defaulton" + require_relative "http1" require_relative "http2" module Async module HTTP module Protocol - # HTTP is an http:// server that auto-selects HTTP/1.1 or HTTP/2 by detecting the HTTP/2 - # connection preface. - module HTTP + # HTTP is an http:// server that auto-selects HTTP/1.1 or HTTP/2 by detecting the HTTP/2 connection preface. + class HTTP HTTP2_PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" HTTP2_PREFACE_SIZE = HTTP2_PREFACE.bytesize - def self.protocol_for(stream) + # Create a new HTTP protocol instance. + # + # @parameter http1 [HTTP1] The HTTP/1 protocol instance. + # @parameter http2 [HTTP2] The HTTP/2 protocol instance. + def initialize(http1: HTTP1, http2: HTTP2) + @http1 = http1 + @http2 = http2 + end + + # Determine if the inbound connection is HTTP/1 or HTTP/2. + # + # @parameter stream [IO::Stream] The stream to detect the protocol for. + # @returns [Class] The protocol class to use. + def protocol_for(stream) # Detect HTTP/2 connection preface # https://www.rfc-editor.org/rfc/rfc9113.html#section-3.4 preface = stream.peek do |read_buffer| @@ -29,27 +43,35 @@ def self.protocol_for(stream) end if preface == HTTP2_PREFACE - HTTP2 + @http2 else - HTTP1 + @http1 end end - # Only inbound connections can detect HTTP1 vs HTTP2 for http://. - # Outbound connections default to HTTP1. - def self.client(peer, **options) - HTTP1.client(peer, **options) + # Create a client for an outbound connection. Defaults to HTTP/1 for plaintext connections. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the protocol, keyed by protocol class. + def client(peer, **options) + options = options[@http1] || {} + + return @http1.client(peer, **options) end - def self.server(peer, **options) - stream = ::IO::Stream(peer) + # Create a server for an inbound connection. Able to detect HTTP1 and HTTP2. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the protocol, keyed by protocol class. + def server(peer, **options) + stream = IO::Stream(peer) + protocol = protocol_for(stream) + options = options[protocol] || {} - return protocol_for(stream).server(stream, **options) + return protocol.server(stream, **options) end - def self.names - ["h2", "http/1.1", "http/1.0"] - end + extend Defaulton end end end diff --git a/lib/async/http/protocol/http1.rb b/lib/async/http/protocol/http1.rb index 2d7b23a..3edc844 100644 --- a/lib/async/http/protocol/http1.rb +++ b/lib/async/http/protocol/http1.rb @@ -4,6 +4,8 @@ # Copyright, 2017-2024, by Samuel Williams. # Copyright, 2024, by Thomas Morgan. +require_relative "configurable" + require_relative "http1/client" require_relative "http1/server" @@ -13,28 +15,41 @@ module Async module HTTP module Protocol module HTTP1 + extend Configurable + VERSION = "HTTP/1.1" + # @returns [Boolean] Whether the protocol supports bidirectional communication. def self.bidirectional? true end + # @returns [Boolean] Whether the protocol supports trailers. def self.trailer? true end - def self.client(peer) + # Create a client for an outbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the client instance. + def self.client(peer, **options) stream = ::IO::Stream(peer) - return HTTP1::Client.new(stream, VERSION) + return HTTP1::Client.new(stream, VERSION, **options) end - def self.server(peer) + # Create a server for an inbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the server instance. + def self.server(peer, **options) stream = ::IO::Stream(peer) - return HTTP1::Server.new(stream, VERSION) + return HTTP1::Server.new(stream, VERSION, **options) end + # @returns [Array] The names of the supported protocol. def self.names ["http/1.1", "http/1.0"] end diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index 1ceed5d..7ba94c5 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -14,9 +14,10 @@ module HTTP module Protocol module HTTP1 class Connection < ::Protocol::HTTP1::Connection - def initialize(stream, version) - super(stream) + def initialize(stream, version, **options) + super(stream, **options) + # On the client side, we need to send the HTTP version with the initial request. On the server side, there are some scenarios (bad request) where we don't know the request version. In those cases, we use this value, which is either hard coded based on the protocol being used, OR could be negotiated during the connection setup (e.g. ALPN). @version = version end diff --git a/lib/async/http/protocol/http10.rb b/lib/async/http/protocol/http10.rb index e37308b..6920b77 100755 --- a/lib/async/http/protocol/http10.rb +++ b/lib/async/http/protocol/http10.rb @@ -10,28 +10,41 @@ module Async module HTTP module Protocol module HTTP10 + extend Configurable + VERSION = "HTTP/1.0" + # @returns [Boolean] Whether the protocol supports bidirectional communication. def self.bidirectional? false end + # @returns [Boolean] Whether the protocol supports trailers. def self.trailer? false end - def self.client(peer) + # Create a client for an outbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the client instance. + def self.client(peer, **options) stream = ::IO::Stream(peer) - return HTTP1::Client.new(stream, VERSION) + return HTTP1::Client.new(stream, VERSION, **options) end - def self.server(peer) + # Create a server for an inbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the server instance. + def self.server(peer, **options) stream = ::IO::Stream(peer) - return HTTP1::Server.new(stream, VERSION) + return HTTP1::Server.new(stream, VERSION, **options) end + # @returns [Array] The names of the supported protocol. def self.names ["http/1.0"] end diff --git a/lib/async/http/protocol/http11.rb b/lib/async/http/protocol/http11.rb index b29f246..814f7e3 100644 --- a/lib/async/http/protocol/http11.rb +++ b/lib/async/http/protocol/http11.rb @@ -11,28 +11,41 @@ module Async module HTTP module Protocol module HTTP11 + extend Configurable + VERSION = "HTTP/1.1" + # @returns [Boolean] Whether the protocol supports bidirectional communication. def self.bidirectional? true end + # @returns [Boolean] Whether the protocol supports trailers. def self.trailer? true end - def self.client(peer) + # Create a client for an outbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the client instance. + def self.client(peer, **options) stream = ::IO::Stream(peer) - return HTTP1::Client.new(stream, VERSION) + return HTTP1::Client.new(stream, VERSION, **options) end - def self.server(peer) + # Create a server for an inbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the server instance. + def self.server(peer, **options) stream = ::IO::Stream(peer) - return HTTP1::Server.new(stream, VERSION) + return HTTP1::Server.new(stream, VERSION, **options) end + # @returns [Array] The names of the supported protocol. def self.names ["http/1.1"] end diff --git a/lib/async/http/protocol/http2.rb b/lib/async/http/protocol/http2.rb index e7a5420..e9a8a84 100644 --- a/lib/async/http/protocol/http2.rb +++ b/lib/async/http/protocol/http2.rb @@ -4,6 +4,8 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2024, by Thomas Morgan. +require_relative "configurable" + require_relative "http2/client" require_relative "http2/server" @@ -13,16 +15,21 @@ module Async module HTTP module Protocol module HTTP2 + extend Configurable + VERSION = "HTTP/2" + # @returns [Boolean] Whether the protocol supports bidirectional communication. def self.bidirectional? true end + # @returns [Boolean] Whether the protocol supports trailers. def self.trailer? true end + # The default settings for the client. CLIENT_SETTINGS = { ::Protocol::HTTP2::Settings::ENABLE_PUSH => 0, ::Protocol::HTTP2::Settings::MAXIMUM_FRAME_SIZE => 0x100000, @@ -30,6 +37,7 @@ def self.trailer? ::Protocol::HTTP2::Settings::NO_RFC7540_PRIORITIES => 1, } + # The default settings for the server. SERVER_SETTINGS = { # We choose a lower maximum concurrent streams to avoid overloading a single connection/thread. ::Protocol::HTTP2::Settings::MAXIMUM_CONCURRENT_STREAMS => 128, @@ -39,7 +47,11 @@ def self.trailer? ::Protocol::HTTP2::Settings::NO_RFC7540_PRIORITIES => 1, } - def self.client(peer, settings = CLIENT_SETTINGS) + # Create a client for an outbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the client instance. + def self.client(peer, settings: CLIENT_SETTINGS) stream = ::IO::Stream(peer) client = Client.new(stream) @@ -49,7 +61,11 @@ def self.client(peer, settings = CLIENT_SETTINGS) return client end - def self.server(peer, settings = SERVER_SETTINGS) + # Create a server for an inbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the server instance. + def self.server(peer, settings: SERVER_SETTINGS) stream = ::IO::Stream(peer) server = Server.new(stream) @@ -59,6 +75,7 @@ def self.server(peer, settings = SERVER_SETTINGS) return server end + # @returns [Array] The names of the supported protocol. def self.names ["h2"] end diff --git a/lib/async/http/protocol/http2/connection.rb b/lib/async/http/protocol/http2/connection.rb index 7757703..e2c2876 100644 --- a/lib/async/http/protocol/http2/connection.rb +++ b/lib/async/http/protocol/http2/connection.rb @@ -26,7 +26,7 @@ module HTTP2 TRAILER = "trailer".freeze module Connection - def initialize(*) + def initialize(...) super @reader = nil diff --git a/lib/async/http/protocol/https.rb b/lib/async/http/protocol/https.rb index a79784a..1fa4535 100644 --- a/lib/async/http/protocol/https.rb +++ b/lib/async/http/protocol/https.rb @@ -4,16 +4,18 @@ # Copyright, 2018-2024, by Samuel Williams. # Copyright, 2019, by Brian Morearty. +require_relative "defaulton" + require_relative "http10" require_relative "http11" - require_relative "http2" module Async module HTTP module Protocol # A server that supports both HTTP1.0 and HTTP1.1 semantics by detecting the version of the request. - module HTTPS + class HTTPS + # The protocol classes for each supported protocol. HANDLERS = { "h2" => HTTP2, "http/1.1" => HTTP11, @@ -21,7 +23,23 @@ module HTTPS nil => HTTP11, } - def self.protocol_for(peer) + def initialize(handlers = HANDLERS, **options) + @handlers = handlers + @options = options + end + + def add(name, protocol, **options) + @handlers[name] = protocol + @options[protocol] = options + end + + # Determine the protocol of the peer and return the appropriate protocol class. + # + # Use TLS Application Layer Protocol Negotiation (ALPN) to determine the protocol. + # + # @parameter peer [IO] The peer to communicate with. + # @returns [Class] The protocol class to use. + def protocol_for(peer) # alpn_protocol is only available if openssl v1.0.2+ name = peer.alpn_protocol @@ -34,18 +52,34 @@ def self.protocol_for(peer) end end - def self.client(peer) - protocol_for(peer).client(peer) + # Create a client for an outbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the client instance. + def client(peer, **options) + protocol = protocol_for(peer) + options = options[protocol] || {} + + protocol.client(peer, **options) end - def self.server(peer) - protocol_for(peer).server(peer) + # Create a server for an inbound connection. + # + # @parameter peer [IO] The peer to communicate with. + # @parameter options [Hash] Options to pass to the server instance. + def server(peer, **options) + protocol = protocol_for(peer) + options = options[protocol] || {} + + protocol.server(peer, **options) end - # Supported Application Layer Protocol Negotiation names: - def self.names - HANDLERS.keys.compact + # @returns [Array] The names of the supported protocol, used for Application Layer Protocol Negotiation (ALPN). + def names + @handlers.keys.compact end + + extend Defaulton end end end diff --git a/releases.md b/releases.md index 59bd24c..3dd42a1 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,66 @@ # Releases +## Unreleased + +### Support custom protocols with options + +{ruby Async::HTTP::Protocol} contains classes for specific protocols, e.g. {ruby Async::HTTP::Protocol::HTTP1} and {ruby Async::HTTP::Protocol::HTTP2}. It also contains classes for aggregating protocols, e.g. {ruby Async::HTTP::Protocol::HTTP} and {ruby Async::HTTP::Protocol::HTTPS}. They serve as factories for creating client and server instances. + +These classes are now configurable with various options, which are passed as keyword arguments to the relevant connection classes. For example, to configure an HTTP/1.1 protocol without keep-alive: + +```ruby +protocol = Async::HTTP::Protocol::HTTP1.new(persistent: false, maximum_line_length: 32) +endpoint = Async::HTTP::Endpoint.parse("http://localhost:9292", protocol: protocol) +server = Async::HTTP::Server.for(endpoint) do |request| + Protocol::HTTP::Response[200, {}, ["Hello, world"]] +end.run +``` + +Making a request to the server will now close the connection after the response is received: + +``` +> curl -v http://localhost:9292 +* Host localhost:9292 was resolved. +* IPv6: ::1 +* IPv4: 127.0.0.1 +* Trying [::1]:9292... +* Connected to localhost (::1) port 9292 +* using HTTP/1.x +> GET / HTTP/1.1 +> Host: localhost:9292 +> User-Agent: curl/8.12.1 +> Accept: */* +> +* Request completely sent off +< HTTP/1.1 200 OK +< connection: close +< content-length: 12 +< +* shutting down connection #0 +Hello, world +``` + +In addition, any line longer than 32 bytes will be rejected: + +``` +curl -v http://localhost:9292/012345678901234567890123456789012 +* Host localhost:9292 was resolved. +* IPv6: ::1 +* IPv4: 127.0.0.1 +* Trying [::1]:9292... +* Connected to localhost (::1) port 9292 +* using HTTP/1.x +> GET /012345678901234567890123456789012 HTTP/1.1 +> Host: localhost:9292 +> User-Agent: curl/8.12.1 +> Accept: */* +> +* Request completely sent off +* Empty reply from server +* shutting down connection #0 +curl: (52) Empty reply from server +``` + ## v0.87.0 ### Unify HTTP/1 and HTTP/2 `CONNECT` semantics diff --git a/test/async/http/protocol/http.rb b/test/async/http/protocol/http.rb index eaaa947..7cb37cb 100755 --- a/test/async/http/protocol/http.rb +++ b/test/async/http/protocol/http.rb @@ -8,6 +8,38 @@ require "async/http/a_protocol" describe Async::HTTP::Protocol::HTTP do + let(:protocol) {subject.default} + + with ".default" do + it "has a default instance" do + expect(protocol).to be_a Async::HTTP::Protocol::HTTP + end + end + + with "#protocol_for" do + let(:buffer) {StringIO.new} + + it "it can detect http/1.1" do + buffer.write("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n") + buffer.rewind + + stream = IO::Stream(buffer) + + expect(protocol.protocol_for(stream)).to be == Async::HTTP::Protocol::HTTP1 + end + + it "it can detect http/2" do + # This special preface is used to indicate that the client would like to use HTTP/2. + # https://www.rfc-editor.org/rfc/rfc7540.html#section-3.5 + buffer.write("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") + buffer.rewind + + stream = IO::Stream(buffer) + + expect(protocol.protocol_for(stream)).to be == Async::HTTP::Protocol::HTTP2 + end + end + with "server" do include Sus::Fixtures::Async::HTTP::ServerContext let(:protocol) {subject} diff --git a/test/async/http/protocol/http1.rb b/test/async/http/protocol/http1.rb new file mode 100644 index 0000000..7bfa945 --- /dev/null +++ b/test/async/http/protocol/http1.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2024, by Thomas Morgan. +# Copyright, 2024, by Samuel Williams. + +require "async/http/protocol/http" +require "async/http/a_protocol" + +describe Async::HTTP::Protocol::HTTP1 do + with ".new" do + it "can configure the protocol" do + protocol = subject.new( + persistent: false, + maximum_line_length: 4096, + ) + + expect(protocol.options).to have_keys( + persistent: be == false, + maximum_line_length: be == 4096, + ) + end + end +end diff --git a/test/async/http/protocol/https.rb b/test/async/http/protocol/https.rb new file mode 100644 index 0000000..0624748 --- /dev/null +++ b/test/async/http/protocol/https.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2025, by Samuel Williams. + +require "async/http/protocol/https" +require "async/http/a_protocol" + +describe Async::HTTP::Protocol::HTTPS do + let(:protocol) {subject.default} + + with ".default" do + it "has a default instance" do + expect(protocol).to be_a Async::HTTP::Protocol::HTTPS + end + + it "supports http/1.0" do + expect(protocol.names).to be(:include?, "http/1.0") + end + + it "supports http/1.1" do + expect(protocol.names).to be(:include?, "http/1.1") + end + + it "supports h2" do + expect(protocol.names).to be(:include?, "h2") + end + end + + with "#protocol_for" do + let(:buffer) {StringIO.new} + + it "can detect http/1.0" do + stream = IO::Stream(buffer) + expect(stream).to receive(:alpn_protocol).and_return("http/1.0") + + expect(protocol.protocol_for(stream)).to be == Async::HTTP::Protocol::HTTP10 + end + + it "it can detect http/1.1" do + stream = IO::Stream(buffer) + expect(stream).to receive(:alpn_protocol).and_return("http/1.1") + + expect(protocol.protocol_for(stream)).to be == Async::HTTP::Protocol::HTTP11 + end + + it "it can detect http/2" do + stream = IO::Stream(buffer) + expect(stream).to receive(:alpn_protocol).and_return("h2") + + expect(protocol.protocol_for(stream)).to be == Async::HTTP::Protocol::HTTP2 + end + end +end From 07cfd2e14ff6e467b1640c1e50bd41b303ee986a Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 13 Mar 2025 23:19:19 +1300 Subject: [PATCH 122/125] Bump minor version. --- lib/async/http/version.rb | 2 +- readme.md | 8 ++-- releases.md | 78 +++++++++++++++++++-------------------- 3 files changed, 42 insertions(+), 46 deletions(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 9bfe6ba..643dadc 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.87.0" + VERSION = "0.88.0" end end diff --git a/readme.md b/readme.md index f15c6f4..2ceb4fd 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,10 @@ Please see the [project documentation](https://socketry.github.io/async-http/) f Please see the [project releases](https://socketry.github.io/async-http/releases/index) for all releases. +### v0.88.0 + + - [Support custom protocols with options](https://socketry.github.io/async-http/releases/index#support-custom-protocols-with-options) + ### v0.87.0 - [Unify HTTP/1 and HTTP/2 `CONNECT` semantics](https://socketry.github.io/async-http/releases/index#unify-http/1-and-http/2-connect-semantics) @@ -55,10 +59,6 @@ Please see the [project releases](https://socketry.github.io/async-http/releases - [`Async::HTTP::Internet` accepts keyword arguments](https://socketry.github.io/async-http/releases/index#async::http::internet-accepts-keyword-arguments) -### v0.73.0 - - - [Update support for `interim_response`](https://socketry.github.io/async-http/releases/index#update-support-for-interim_response) - ## See Also - [benchmark-http](https://github.com/socketry/benchmark-http) — A benchmarking tool to report on web server concurrency. diff --git a/releases.md b/releases.md index 3dd42a1..3b58d50 100644 --- a/releases.md +++ b/releases.md @@ -1,6 +1,6 @@ # Releases -## Unreleased +## v0.88.0 ### Support custom protocols with options @@ -8,7 +8,7 @@ These classes are now configurable with various options, which are passed as keyword arguments to the relevant connection classes. For example, to configure an HTTP/1.1 protocol without keep-alive: -```ruby +``` ruby protocol = Async::HTTP::Protocol::HTTP1.new(persistent: false, maximum_line_length: 32) endpoint = Async::HTTP::Endpoint.parse("http://localhost:9292", protocol: protocol) server = Async::HTTP::Server.for(endpoint) do |request| @@ -18,48 +18,44 @@ end.run Making a request to the server will now close the connection after the response is received: -``` -> curl -v http://localhost:9292 -* Host localhost:9292 was resolved. -* IPv6: ::1 -* IPv4: 127.0.0.1 -* Trying [::1]:9292... -* Connected to localhost (::1) port 9292 -* using HTTP/1.x -> GET / HTTP/1.1 -> Host: localhost:9292 -> User-Agent: curl/8.12.1 -> Accept: */* -> -* Request completely sent off -< HTTP/1.1 200 OK -< connection: close -< content-length: 12 -< -* shutting down connection #0 -Hello, world -``` + > curl -v http://localhost:9292 + * Host localhost:9292 was resolved. + * IPv6: ::1 + * IPv4: 127.0.0.1 + * Trying [::1]:9292... + * Connected to localhost (::1) port 9292 + * using HTTP/1.x + > GET / HTTP/1.1 + > Host: localhost:9292 + > User-Agent: curl/8.12.1 + > Accept: */* + > + * Request completely sent off + < HTTP/1.1 200 OK + < connection: close + < content-length: 12 + < + * shutting down connection #0 + Hello, world In addition, any line longer than 32 bytes will be rejected: -``` -curl -v http://localhost:9292/012345678901234567890123456789012 -* Host localhost:9292 was resolved. -* IPv6: ::1 -* IPv4: 127.0.0.1 -* Trying [::1]:9292... -* Connected to localhost (::1) port 9292 -* using HTTP/1.x -> GET /012345678901234567890123456789012 HTTP/1.1 -> Host: localhost:9292 -> User-Agent: curl/8.12.1 -> Accept: */* -> -* Request completely sent off -* Empty reply from server -* shutting down connection #0 -curl: (52) Empty reply from server -``` + curl -v http://localhost:9292/012345678901234567890123456789012 + * Host localhost:9292 was resolved. + * IPv6: ::1 + * IPv4: 127.0.0.1 + * Trying [::1]:9292... + * Connected to localhost (::1) port 9292 + * using HTTP/1.x + > GET /012345678901234567890123456789012 HTTP/1.1 + > Host: localhost:9292 + > User-Agent: curl/8.12.1 + > Accept: */* + > + * Request completely sent off + * Empty reply from server + * shutting down connection #0 + curl: (52) Empty reply from server ## v0.87.0 From 79b4f05d915ce6f7a72e4256312857937bc8b627 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 28 Apr 2025 10:08:58 +0200 Subject: [PATCH 123/125] Fix unused variable warnings ``` async-http-0.86.0/lib/async/http/protocol/http1/server.rb:55: warning: assigned but unused variable - error async-http-0.86.0/lib/async/http/protocol/http2/connection.rb:99: warning: assigned but unused variable - ignored_error ``` --- lib/async/http/protocol/http1/server.rb | 2 +- lib/async/http/protocol/http2/connection.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 2ebf1ca..1c0e45f 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -52,7 +52,7 @@ def next_request end return request - rescue ::Protocol::HTTP1::BadRequest => error + rescue ::Protocol::HTTP1::BadRequest fail_request(400) # Conceivably we could retry here, but we don't really know how bad the error is, so it's better to just fail: raise diff --git a/lib/async/http/protocol/http2/connection.rb b/lib/async/http/protocol/http2/connection.rb index e2c2876..1e84d23 100644 --- a/lib/async/http/protocol/http2/connection.rb +++ b/lib/async/http/protocol/http2/connection.rb @@ -94,7 +94,7 @@ def read_in_background(parent: Task.current) # Error is raised if a response is actively reading from the # connection. The connection is silently closed if GOAWAY is # received outside the request/response cycle. - rescue SocketError, IOError, EOFError, Errno::ECONNRESET, Errno::EPIPE => ignored_error + rescue SocketError, IOError, EOFError, Errno::ECONNRESET, Errno::EPIPE # Ignore. rescue => error # Every other error. From f89a3517109d8b58c2d37fb93fa6031170b46745 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 28 Apr 2025 20:31:32 +0900 Subject: [PATCH 124/125] Modernize code + drop support for Ruby v3.1. --- .github/workflows/test-external.yaml | 1 - .github/workflows/test.yaml | 1 - .rubocop.yml | 3 +++ async-http.gemspec | 4 ++-- bake/async/http/h2spec.rb | 2 +- examples/download/chunked.rb | 2 +- examples/google/search.rb | 2 +- examples/licenses/list.rb | 2 +- examples/stream/stop.rb | 2 +- lib/async/http/client.rb | 2 +- lib/async/http/endpoint.rb | 2 +- lib/async/http/protocol/http.rb | 2 +- lib/async/http/protocol/http1.rb | 2 +- lib/async/http/protocol/http1/connection.rb | 2 +- lib/async/http/protocol/http1/server.rb | 3 ++- lib/async/http/protocol/http10.rb | 2 +- lib/async/http/protocol/http11.rb | 2 +- lib/async/http/protocol/http2.rb | 2 +- lib/async/http/protocol/http2/client.rb | 2 +- lib/async/http/protocol/http2/connection.rb | 3 ++- lib/async/http/protocol/http2/server.rb | 2 +- lib/async/http/protocol/http2/stream.rb | 2 +- lib/async/http/protocol/https.rb | 2 +- lib/async/http/server.rb | 2 +- lib/async/http/version.rb | 2 +- license.md | 1 + test/async/http/protocol/http.rb | 2 +- test/async/http/protocol/http1.rb | 3 +-- test/async/http/protocol/http2.rb | 2 +- 29 files changed, 32 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test-external.yaml b/.github/workflows/test-external.yaml index c9cc200..217a86a 100644 --- a/.github/workflows/test-external.yaml +++ b/.github/workflows/test-external.yaml @@ -20,7 +20,6 @@ jobs: - macos ruby: - - "3.1" - "3.2" - "3.3" - "3.4" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4b87012..5347085 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,7 +23,6 @@ jobs: - macos ruby: - - "3.1" - "3.2" - "3.3" - "3.4" diff --git a/.rubocop.yml b/.rubocop.yml index 3b8d476..573c239 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -45,6 +45,9 @@ Layout/EmptyLinesAroundClassBody: Layout/EmptyLinesAroundModuleBody: Enabled: true +Layout/EmptyLineAfterMagicComment: + Enabled: true + Style/FrozenStringLiteralComment: Enabled: true diff --git a/async-http.gemspec b/async-http.gemspec index 6565546..6f2ae8f 100644 --- a/async-http.gemspec +++ b/async-http.gemspec @@ -7,7 +7,7 @@ Gem::Specification.new do |spec| spec.version = Async::HTTP::VERSION spec.summary = "A HTTP client and server library." - spec.authors = ["Samuel Williams", "Brian Morearty", "Bruno Sutic", "Janko Marohnić", "Thomas Morgan", "Adam Daniels", "Igor Sidorov", "Anton Zhuravsky", "Cyril Roelandt", "Denis Talakevich", "Hal Brodigan", "Ian Ker-Seymer", "Josh Huber", "Marco Concetto Rudilosso", "Olle Jonsson", "Orgad Shaneh", "Sam Shadwell", "Stefan Wrobel", "Tim Meusel", "Trevor Turk", "Viacheslav Koval", "dependabot[bot]"] + spec.authors = ["Samuel Williams", "Brian Morearty", "Bruno Sutic", "Janko Marohnić", "Thomas Morgan", "Adam Daniels", "Igor Sidorov", "Anton Zhuravsky", "Cyril Roelandt", "Denis Talakevich", "Hal Brodigan", "Ian Ker-Seymer", "Jean Boussier", "Josh Huber", "Marco Concetto Rudilosso", "Olle Jonsson", "Orgad Shaneh", "Sam Shadwell", "Stefan Wrobel", "Tim Meusel", "Trevor Turk", "Viacheslav Koval", "dependabot[bot]"] spec.license = "MIT" spec.cert_chain = ["release.cert"] @@ -22,7 +22,7 @@ Gem::Specification.new do |spec| spec.files = Dir.glob(["{bake,lib}/**/*", "*.md"], File::FNM_DOTMATCH, base: __dir__) - spec.required_ruby_version = ">= 3.1" + spec.required_ruby_version = ">= 3.2" spec.add_dependency "async", ">= 2.10.2" spec.add_dependency "async-pool", "~> 0.9" diff --git a/bake/async/http/h2spec.rb b/bake/async/http/h2spec.rb index 9aec9e3..0212e32 100644 --- a/bake/async/http/h2spec.rb +++ b/bake/async/http/h2spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2019-2024, by Samuel Williams. +# Copyright, 2019-2025, by Samuel Williams. def build # Fetch the code: diff --git a/examples/download/chunked.rb b/examples/download/chunked.rb index 1a40943..14daa87 100755 --- a/examples/download/chunked.rb +++ b/examples/download/chunked.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2024, by Samuel Williams. +# Copyright, 2020-2025, by Samuel Williams. require "async" require "async/clock" diff --git a/examples/google/search.rb b/examples/google/search.rb index bbe1ec6..956548a 100755 --- a/examples/google/search.rb +++ b/examples/google/search.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2019-2024, by Samuel Williams. +# Copyright, 2019-2025, by Samuel Williams. require "async" require "async/clock" diff --git a/examples/licenses/list.rb b/examples/licenses/list.rb index 3e03524..87da93e 100755 --- a/examples/licenses/list.rb +++ b/examples/licenses/list.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2024, by Samuel Williams. +# Copyright, 2020-2025, by Samuel Williams. require "csv" require "json" diff --git a/examples/stream/stop.rb b/examples/stream/stop.rb index a4e1cc1..422a978 100644 --- a/examples/stream/stop.rb +++ b/examples/stream/stop.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2020-2024, by Samuel Williams. +# Copyright, 2020-2025, by Samuel Williams. require "async" require "async/http/internet" diff --git a/lib/async/http/client.rb b/lib/async/http/client.rb index 743cba0..8911747 100755 --- a/lib/async/http/client.rb +++ b/lib/async/http/client.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2024, by Samuel Williams. +# Copyright, 2017-2025, by Samuel Williams. # Copyright, 2022, by Ian Ker-Seymer. require "io/endpoint" diff --git a/lib/async/http/endpoint.rb b/lib/async/http/endpoint.rb index 4a67f72..57122ef 100644 --- a/lib/async/http/endpoint.rb +++ b/lib/async/http/endpoint.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2019-2024, by Samuel Williams. +# Copyright, 2019-2025, by Samuel Williams. # Copyright, 2021-2022, by Adam Daniels. # Copyright, 2024, by Thomas Morgan. # Copyright, 2024, by Igor Sidorov. diff --git a/lib/async/http/protocol/http.rb b/lib/async/http/protocol/http.rb index 3864222..97d8f37 100644 --- a/lib/async/http/protocol/http.rb +++ b/lib/async/http/protocol/http.rb @@ -2,7 +2,7 @@ # Released under the MIT License. # Copyright, 2024, by Thomas Morgan. -# Copyright, 2024, by Samuel Williams. +# Copyright, 2024-2025, by Samuel Williams. require_relative "defaulton" diff --git a/lib/async/http/protocol/http1.rb b/lib/async/http/protocol/http1.rb index 3edc844..d6061a7 100644 --- a/lib/async/http/protocol/http1.rb +++ b/lib/async/http/protocol/http1.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2024, by Samuel Williams. +# Copyright, 2017-2025, by Samuel Williams. # Copyright, 2024, by Thomas Morgan. require_relative "configurable" diff --git a/lib/async/http/protocol/http1/connection.rb b/lib/async/http/protocol/http1/connection.rb index 7ba94c5..91dbedc 100755 --- a/lib/async/http/protocol/http1/connection.rb +++ b/lib/async/http/protocol/http1/connection.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. require_relative "request" require_relative "response" diff --git a/lib/async/http/protocol/http1/server.rb b/lib/async/http/protocol/http1/server.rb index 1c0e45f..6d90f1f 100644 --- a/lib/async/http/protocol/http1/server.rb +++ b/lib/async/http/protocol/http1/server.rb @@ -1,10 +1,11 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. # Copyright, 2020, by Igor Sidorov. # Copyright, 2023, by Thomas Morgan. # Copyright, 2024, by Anton Zhuravsky. +# Copyright, 2025, by Jean Boussier. require_relative "connection" require_relative "finishable" diff --git a/lib/async/http/protocol/http10.rb b/lib/async/http/protocol/http10.rb index 6920b77..53b1a87 100755 --- a/lib/async/http/protocol/http10.rb +++ b/lib/async/http/protocol/http10.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2024, by Samuel Williams. +# Copyright, 2017-2025, by Samuel Williams. # Copyright, 2024, by Thomas Morgan. require_relative "http1" diff --git a/lib/async/http/protocol/http11.rb b/lib/async/http/protocol/http11.rb index 814f7e3..5cb3ab8 100644 --- a/lib/async/http/protocol/http11.rb +++ b/lib/async/http/protocol/http11.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2024, by Samuel Williams. +# Copyright, 2017-2025, by Samuel Williams. # Copyright, 2018, by Janko Marohnić. # Copyright, 2024, by Thomas Morgan. diff --git a/lib/async/http/protocol/http2.rb b/lib/async/http/protocol/http2.rb index e9a8a84..1a44d7e 100644 --- a/lib/async/http/protocol/http2.rb +++ b/lib/async/http/protocol/http2.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. # Copyright, 2024, by Thomas Morgan. require_relative "configurable" diff --git a/lib/async/http/protocol/http2/client.rb b/lib/async/http/protocol/http2/client.rb index f59b4b6..2fe6bc8 100644 --- a/lib/async/http/protocol/http2/client.rb +++ b/lib/async/http/protocol/http2/client.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. require_relative "connection" require_relative "response" diff --git a/lib/async/http/protocol/http2/connection.rb b/lib/async/http/protocol/http2/connection.rb index 1e84d23..ab56eed 100644 --- a/lib/async/http/protocol/http2/connection.rb +++ b/lib/async/http/protocol/http2/connection.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. # Copyright, 2020, by Bruno Sutic. +# Copyright, 2025, by Jean Boussier. require_relative "stream" diff --git a/lib/async/http/protocol/http2/server.rb b/lib/async/http/protocol/http2/server.rb index 32e2d6e..485a87a 100644 --- a/lib/async/http/protocol/http2/server.rb +++ b/lib/async/http/protocol/http2/server.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. require_relative "connection" require_relative "request" diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index 0ed5d5f..9c174b8 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. # Copyright, 2022, by Marco Concetto Rudilosso. # Copyright, 2023, by Thomas Morgan. diff --git a/lib/async/http/protocol/https.rb b/lib/async/http/protocol/https.rb index 1fa4535..5464d4c 100644 --- a/lib/async/http/protocol/https.rb +++ b/lib/async/http/protocol/https.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. # Copyright, 2019, by Brian Morearty. require_relative "defaulton" diff --git a/lib/async/http/server.rb b/lib/async/http/server.rb index 3fa3ade..76e520e 100755 --- a/lib/async/http/server.rb +++ b/lib/async/http/server.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2024, by Samuel Williams. +# Copyright, 2017-2025, by Samuel Williams. # Copyright, 2019, by Brian Morearty. require "async" diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index 643dadc..d47cd15 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2017-2024, by Samuel Williams. +# Copyright, 2017-2025, by Samuel Williams. module Async module HTTP diff --git a/license.md b/license.md index 992c353..014263a 100644 --- a/license.md +++ b/license.md @@ -22,6 +22,7 @@ Copyright, 2023, by dependabot[bot]. Copyright, 2023, by Josh Huber. Copyright, 2024, by Anton Zhuravsky. Copyright, 2024, by Hal Brodigan. +Copyright, 2025, by Jean Boussier. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/test/async/http/protocol/http.rb b/test/async/http/protocol/http.rb index 7cb37cb..4a151a1 100755 --- a/test/async/http/protocol/http.rb +++ b/test/async/http/protocol/http.rb @@ -2,7 +2,7 @@ # Released under the MIT License. # Copyright, 2024, by Thomas Morgan. -# Copyright, 2024, by Samuel Williams. +# Copyright, 2024-2025, by Samuel Williams. require "async/http/protocol/http" require "async/http/a_protocol" diff --git a/test/async/http/protocol/http1.rb b/test/async/http/protocol/http1.rb index 7bfa945..b377d59 100644 --- a/test/async/http/protocol/http1.rb +++ b/test/async/http/protocol/http1.rb @@ -1,8 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2024, by Thomas Morgan. -# Copyright, 2024, by Samuel Williams. +# Copyright, 2025, by Samuel Williams. require "async/http/protocol/http" require "async/http/a_protocol" diff --git a/test/async/http/protocol/http2.rb b/test/async/http/protocol/http2.rb index cd7d308..b33cbd8 100644 --- a/test/async/http/protocol/http2.rb +++ b/test/async/http/protocol/http2.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Released under the MIT License. -# Copyright, 2018-2024, by Samuel Williams. +# Copyright, 2018-2025, by Samuel Williams. require "async/http/protocol/http2" require "async/http/a_protocol" From 58c38c3109fe131c7c621d543f3ac37bba8b66b3 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 28 Apr 2025 20:47:43 +0900 Subject: [PATCH 125/125] Bump minor version. --- lib/async/http/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/http/version.rb b/lib/async/http/version.rb index d47cd15..6538bc5 100644 --- a/lib/async/http/version.rb +++ b/lib/async/http/version.rb @@ -5,6 +5,6 @@ module Async module HTTP - VERSION = "0.88.0" + VERSION = "0.89.0" end end