From e8c9f99aeed4bad2e509b6ca651e939db38ba80b Mon Sep 17 00:00:00 2001 From: nick evans Date: Sat, 22 Mar 2025 14:31:32 -0400 Subject: [PATCH 01/23] =?UTF-8?q?=F0=9F=93=9A=20Add=20docs=20for=20receive?= =?UTF-8?q?r=20thread=20&=20server=20responses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most importantly, this documents the scenarios that need extra care to avoid memory leaks: * Commands such as #list or #fetch can have an enormous number of responses. * Commands such as #fetch can result in an enormous size per response. * Long-lived connections will gradually accumulate unsolicited server responses, especially +EXISTS+, +FETCH+, and +EXPUNGE+ responses. * A buggy or untrusted server could send inappropriate responses, which could be very numerous, very large, and very rapid. --- lib/net/imap.rb | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index ea1bd308a..501ef4ed0 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -106,6 +106,37 @@ module Net # # This script invokes the FETCH command and the SEARCH command concurrently. # + # When running multiple commands, care must be taken to avoid ambiguity. For + # example, SEARCH responses are ambiguous about which command they are + # responding to, so search commands should not run simultaneously, unless the + # server supports +ESEARCH+ {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731] or + # IMAP4rev2[https://www.rfc-editor.org/rfc/rfc9051]. See {RFC9051 + # ยง5.5}[https://www.rfc-editor.org/rfc/rfc9051.html#section-5.5] for + # other examples of command sequences which should not be pipelined. + # + # == Unbounded memory use + # + # Net::IMAP reads server responses in a separate receiver thread per client. + # Unhandled response data is saved to #responses, and response_handlers run + # inside the receiver thread. See the list of methods for {handling server + # responses}[rdoc-ref:Net::IMAP@Handling+server+responses], below. + # + # Because the receiver thread continuously reads and saves new responses, some + # scenarios must be careful to avoid unbounded memory use: + # + # * Commands such as #list or #fetch can have an enormous number of responses. + # * Commands such as #fetch can result in an enormous size per response. + # * Long-lived connections will gradually accumulate unsolicited server + # responses, especially +EXISTS+, +FETCH+, and +EXPUNGE+ responses. + # * A buggy or untrusted server could send inappropriate responses, which + # could be very numerous, very large, and very rapid. + # + # Use paginated or limited versions of commands whenever possible. + # + # Use #add_response_handler to handle responses after each one is received. + # Use #extract_responses, #clear_responses, or #responses (with a block) to + # prune responses. + # # == Errors # # An IMAP server can send three different types of responses to indicate From eddbb18d801f34c9162a4e7483d5ba5fccc30048 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sun, 20 Apr 2025 18:18:23 -0400 Subject: [PATCH 02/23] =?UTF-8?q?=F0=9F=93=9A=20Delete=20backported=20docs?= =?UTF-8?q?=20that=20are=20only=20for=20v0.4+?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 501ef4ed0..898ab90f9 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -134,8 +134,6 @@ module Net # Use paginated or limited versions of commands whenever possible. # # Use #add_response_handler to handle responses after each one is received. - # Use #extract_responses, #clear_responses, or #responses (with a block) to - # prune responses. # # == Errors # From d50e9f31b81a4aa08faa69555fc7dd1df53d280c Mon Sep 17 00:00:00 2001 From: nick evans Date: Sun, 20 Apr 2025 22:41:09 -0400 Subject: [PATCH 03/23] =?UTF-8?q?=E2=9C=85=20Add=20modern=20ruby=20version?= =?UTF-8?q?s=20to=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In prep for the final 0.2.x release, I want to make sure we're testing against all relevant ruby versions. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d959a8cde..e379ab3b7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: name: build (${{ matrix.ruby }} / ${{ matrix.os }}) strategy: matrix: - ruby: [ head, '3.0', '2.7' ] + ruby: [ head, '3.3', '3.2', '3.1', '3.0', '2.7' ] os: [ ubuntu-latest, macos-latest ] experimental: [false] include: From 5a7828e97f74a563d8b1adc497cbb83487e3e0c8 Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 24 Mar 2025 17:35:10 -0400 Subject: [PATCH 04/23] =?UTF-8?q?=F0=9F=8E=A8=20Reformat=20get=5Fresponse?= =?UTF-8?q?=20debug=20trace=20printing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 17f3148cd..831e56e13 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -1308,10 +1308,8 @@ def get_response end end return nil if buff.length == 0 - if @@debug - $stderr.print(buff.gsub(/^/n, "S: ")) - end - return @parser.parse(buff) + $stderr.print(buff.gsub(/^/n, "S: ")) if @@debug + @parser.parse(buff) end def record_response(name, data) From 936ecde3bc2e0446194bf76c231ac562a708c370 Mon Sep 17 00:00:00 2001 From: nick evans Date: Tue, 25 Mar 2025 13:08:21 -0400 Subject: [PATCH 05/23] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Anchor=20literal=20r?= =?UTF-8?q?egexp=20to=20the=20end=20of=20the=20buffer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This also allows us to check against the concatenated buffer, rather than the smaller line buffer. That distinction doesn't really matter now, since we always read an entire line at once. But it will matter if we read partial lines. --- lib/net/imap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 831e56e13..712a29d3f 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -1300,7 +1300,7 @@ def get_response s = @sock.gets(CRLF) break unless s buff.concat(s) - if /\{(\d+)\}\r\n/n =~ s + if /\{(\d+)\}\r\n\z/n =~ buff s = @sock.read($1.to_i) buff.concat(s) else From 2b3fa22e6ae5f64f5dd9ae83aaf7e3c4d878bd4f Mon Sep 17 00:00:00 2001 From: nick evans Date: Tue, 25 Mar 2025 12:45:23 -0400 Subject: [PATCH 06/23] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Extract=20line=20and?= =?UTF-8?q?=20literal=20parts=20of=20get=5Fresponse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IMO, this refactoring makes `get_response` much easier to understand. Which will be useful, because I'm about to complicate it. ๐Ÿ˜‰ --- lib/net/imap.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 712a29d3f..b08268d66 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -1297,12 +1297,9 @@ def get_tagged_response(tag, cmd, timeout = nil) def get_response buff = String.new while true - s = @sock.gets(CRLF) - break unless s - buff.concat(s) + get_response_line(buff) or break if /\{(\d+)\}\r\n\z/n =~ buff - s = @sock.read($1.to_i) - buff.concat(s) + get_response_literal(buff, $1.to_i) or break else break end @@ -1312,6 +1309,18 @@ def get_response @parser.parse(buff) end + def get_response_line(buff) + line = @sock.gets(CRLF) or return + buff << line + end + + def get_response_literal(buff, literal_size) + literal = @sock.read(literal_size) or return + buff << literal + end + + ############################# + def record_response(name, data) unless @responses.has_key?(name) @responses[name] = [] From 549510e0041414466d8e2e9a11217ec236d52d33 Mon Sep 17 00:00:00 2001 From: nick evans Date: Tue, 25 Mar 2025 13:44:50 -0400 Subject: [PATCH 07/23] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Allocate=20string=20?= =?UTF-8?q?literals=20with=20specific=20capacity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We know exactly how much memory we're going to need, so we can allocate this up-front, and save a few malloc/memcpy calls on larger literals. --- lib/net/imap.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index b08268d66..9eb707d86 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -1315,7 +1315,8 @@ def get_response_line(buff) end def get_response_literal(buff, literal_size) - literal = @sock.read(literal_size) or return + literal = String.new(capacity: literal_size) + @sock.read(literal_size, literal) or return buff << literal end From acfedcf9a7f2f6e4da271c11a98044125fd459c3 Mon Sep 17 00:00:00 2001 From: nick evans Date: Tue, 25 Mar 2025 12:46:59 -0400 Subject: [PATCH 08/23] =?UTF-8?q?=F0=9F=8E=A8=20Simplify=20get=5Fresponse?= =?UTF-8?q?=20loop=20further?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 9eb707d86..855548baf 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -1298,11 +1298,8 @@ def get_response buff = String.new while true get_response_line(buff) or break - if /\{(\d+)\}\r\n\z/n =~ buff - get_response_literal(buff, $1.to_i) or break - else - break - end + break unless /\{(\d+)\}\r\n\z/n =~ buff + get_response_literal(buff, $1.to_i) or break end return nil if buff.length == 0 $stderr.print(buff.gsub(/^/n, "S: ")) if @@debug From f9248a9689c169f6fcd0242b829020e01b97d67c Mon Sep 17 00:00:00 2001 From: nick evans Date: Tue, 25 Mar 2025 14:00:40 -0400 Subject: [PATCH 09/23] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Explicitly=20"throw?= =?UTF-8?q?=20:eof"=20for=20EOF=20in=20get=5Fresponse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This feels a lot more self-documenting than returning nil then breaking when nil is returned. Also, it lets me refactor the return values for the get_response_line/get_response_literal methods, or throw from even deeper in the stack. --- lib/net/imap.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 855548baf..0b2d82641 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -1296,10 +1296,12 @@ def get_tagged_response(tag, cmd, timeout = nil) def get_response buff = String.new - while true - get_response_line(buff) or break - break unless /\{(\d+)\}\r\n\z/n =~ buff - get_response_literal(buff, $1.to_i) or break + catch :eof do + while true + get_response_line(buff) + break unless /\{(\d+)\}\r\n\z/n =~ buff + get_response_literal(buff, $1.to_i) + end end return nil if buff.length == 0 $stderr.print(buff.gsub(/^/n, "S: ")) if @@debug @@ -1307,13 +1309,13 @@ def get_response end def get_response_line(buff) - line = @sock.gets(CRLF) or return + line = @sock.gets(CRLF) or throw :eof buff << line end def get_response_literal(buff, literal_size) literal = String.new(capacity: literal_size) - @sock.read(literal_size, literal) or return + @sock.read(literal_size, literal) or throw :eof buff << literal end From 570bd632e61df1084e9923c8e1c4d1810e59ef10 Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 2 Apr 2025 20:50:15 -0400 Subject: [PATCH 10/23] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Extract=20ResponseRe?= =?UTF-8?q?ader=20from=20get=5Fresponse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's nice to extract a little bit of the complexity from the core `Net::IMAP` class. But my primary motivation was so that I could directly test this code quickly and in isolation from needing to simulate a full IMAP connection. --- lib/net/imap.rb | 24 +++---------- lib/net/imap/response_reader.rb | 38 +++++++++++++++++++++ test/net/imap/test_response_reader.rb | 49 +++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 lib/net/imap/response_reader.rb create mode 100644 test/net/imap/test_response_reader.rb diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 0b2d82641..f00491008 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -253,6 +253,8 @@ module Net class IMAP < Protocol VERSION = "0.2.4" + autoload :ResponseReader, File.expand_path("imap/response_reader", __dir__) + include MonitorMixin if defined?(OpenSSL::SSL) include OpenSSL @@ -1144,6 +1146,7 @@ def initialize(host, port_or_options = {}, @idle_response_timeout = options[:idle_response_timeout] || 5 @parser = ResponseParser.new @sock = tcp_socket(@host, @port) + @reader = ResponseReader.new(self, @sock) begin if options[:ssl] start_tls_session(options[:ssl]) @@ -1295,30 +1298,12 @@ def get_tagged_response(tag, cmd, timeout = nil) end def get_response - buff = String.new - catch :eof do - while true - get_response_line(buff) - break unless /\{(\d+)\}\r\n\z/n =~ buff - get_response_literal(buff, $1.to_i) - end - end + buff = @reader.read_response_buffer return nil if buff.length == 0 $stderr.print(buff.gsub(/^/n, "S: ")) if @@debug @parser.parse(buff) end - def get_response_line(buff) - line = @sock.gets(CRLF) or throw :eof - buff << line - end - - def get_response_literal(buff, literal_size) - literal = String.new(capacity: literal_size) - @sock.read(literal_size, literal) or throw :eof - buff << literal - end - ############################# def record_response(name, data) @@ -1498,6 +1483,7 @@ def start_tls_session(params = {}) context.verify_callback = VerifyCallbackProc end @sock = SSLSocket.new(@sock, context) + @reader = ResponseReader.new(self, @sock) @sock.sync_close = true @sock.hostname = @host if @sock.respond_to? :hostname= ssl_socket_connect(@sock, @open_timeout) diff --git a/lib/net/imap/response_reader.rb b/lib/net/imap/response_reader.rb new file mode 100644 index 000000000..57770e3b8 --- /dev/null +++ b/lib/net/imap/response_reader.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Net + class IMAP + # See https://www.rfc-editor.org/rfc/rfc9051#section-2.2.2 + class ResponseReader # :nodoc: + attr_reader :client + + def initialize(client, sock) + @client, @sock = client, sock + end + + def read_response_buffer + buff = String.new + catch :eof do + while true + read_line(buff) + break unless /\{(\d+)\}\r\n\z/n =~ buff + read_literal(buff, $1.to_i) + end + end + buff + end + + private + + def read_line(buff) + buff << (@sock.gets(CRLF) or throw :eof) + end + + def read_literal(buff, literal_size) + literal = String.new(capacity: literal_size) + buff << (@sock.read(literal_size, literal) or throw :eof) + end + + end + end +end diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb new file mode 100644 index 000000000..c24f1269d --- /dev/null +++ b/test/net/imap/test_response_reader.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require "net/imap" +require "stringio" +require "test/unit" + +class ResponseReaderTest < Test::Unit::TestCase + def setup + Net::IMAP.config.reset + end + + class FakeClient + def config = @config ||= Net::IMAP.config.new + end + + def literal(str) = "{#{str.bytesize}}\r\n#{str}" + + test "#read_response_buffer" do + client = FakeClient.new + aaaaaaaaa = "a" * (20 << 10) + many_crs = "\r" * 1000 + many_crlfs = "\r\n" * 500 + simple = "* OK greeting\r\n" + long_line = "tag ok #{aaaaaaaaa} #{aaaaaaaaa}\r\n" + literal_aaaa = "* fake #{literal aaaaaaaaa}\r\n" + literal_crlf = "tag ok #{literal many_crlfs} #{literal many_crlfs}\r\n" + illegal_crs = "tag ok #{many_crs} #{many_crs}\r\n" + illegal_lfs = "tag ok #{literal "\r"}\n#{literal "\r"}\n\r\n" + io = StringIO.new([ + simple, + long_line, + literal_aaaa, + literal_crlf, + illegal_crs, + illegal_lfs, + simple, + ].join) + rcvr = Net::IMAP::ResponseReader.new(client, io) + assert_equal simple, rcvr.read_response_buffer.to_str + assert_equal long_line, rcvr.read_response_buffer.to_str + assert_equal literal_aaaa, rcvr.read_response_buffer.to_str + assert_equal literal_crlf, rcvr.read_response_buffer.to_str + assert_equal illegal_crs, rcvr.read_response_buffer.to_str + assert_equal illegal_lfs, rcvr.read_response_buffer.to_str + assert_equal simple, rcvr.read_response_buffer.to_str + assert_equal "", rcvr.read_response_buffer.to_str + end + +end From 3d7271044d5f66272a6a44c54dfb1f5d596a8489 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sat, 22 Mar 2025 15:33:02 -0400 Subject: [PATCH 11/23] =?UTF-8?q?=E2=9C=85=20Add=20tests=20for=20#add=5Fre?= =?UTF-8?q?sponse=5Fhandler?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There weren't any response_handler tests at all, prior to this! --- test/net/imap/test_imap_response_handlers.rb | 53 ++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 test/net/imap/test_imap_response_handlers.rb diff --git a/test/net/imap/test_imap_response_handlers.rb b/test/net/imap/test_imap_response_handlers.rb new file mode 100644 index 000000000..900c2ebca --- /dev/null +++ b/test/net/imap/test_imap_response_handlers.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require "net/imap" +require "test/unit" +require_relative "fake_server" + +class IMAPResponseHandlersTest < Test::Unit::TestCase + include Net::IMAP::FakeServer::TestHelper + + def setup + Net::IMAP.config.reset + @do_not_reverse_lookup = Socket.do_not_reverse_lookup + Socket.do_not_reverse_lookup = true + @threads = [] + end + + def teardown + if !@threads.empty? + assert_join_threads(@threads) + end + ensure + Socket.do_not_reverse_lookup = @do_not_reverse_lookup + end + + test "#add_response_handlers" do + responses = [] + with_fake_server do |server, imap| + server.on("NOOP") do |resp| + 3.times do resp.untagged("#{_1 + 1} EXPUNGE") end + resp.done_ok + end + + assert_equal 0, imap.response_handlers.length + imap.add_response_handler do responses << [:block, _1] end + assert_equal 1, imap.response_handlers.length + imap.add_response_handler(->{ responses << [:proc, _1] }) + assert_equal 2, imap.response_handlers.length + + imap.noop + assert_pattern do + responses => [ + [:block, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 1]], + [:proc, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 1]], + [:block, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 2]], + [:proc, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 2]], + [:block, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 3]], + [:proc, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 3]], + ] + end + end + end + +end From 202df6e9ba622f3a7984d42c1e1251be12ff1084 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sat, 19 Apr 2025 17:39:14 -0400 Subject: [PATCH 12/23] =?UTF-8?q?=E2=9C=85=20Fix=20backport=20compatibilit?= =?UTF-8?q?y=20with=20ruby=202.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/net/imap/test_response_reader.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb index c24f1269d..319dece22 100644 --- a/test/net/imap/test_response_reader.rb +++ b/test/net/imap/test_response_reader.rb @@ -10,10 +10,10 @@ def setup end class FakeClient - def config = @config ||= Net::IMAP.config.new + def config; @config ||= Net::IMAP.config.new end end - def literal(str) = "{#{str.bytesize}}\r\n#{str}" + def literal(str) "{#{str.bytesize}}\r\n#{str}" end test "#read_response_buffer" do client = FakeClient.new From 527568a8cab481c9d0ff19e95640bc62ffd007e7 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sat, 22 Mar 2025 14:53:00 -0400 Subject: [PATCH 13/23] =?UTF-8?q?=E2=9C=A8=20Add=20`response=5Fhandlers`?= =?UTF-8?q?=20kwarg=20to=20Net::IMAP.new?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This ensures every server response is handled, including the greeting. --- lib/net/imap.rb | 15 ++++++++ test/net/imap/test_imap_response_handlers.rb | 37 ++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 898ab90f9..17f3148cd 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -134,6 +134,8 @@ module Net # Use paginated or limited versions of commands whenever possible. # # Use #add_response_handler to handle responses after each one is received. + # Use the +response_handlers+ argument to ::new to assign response handlers + # before the receiver thread is started. # # == Errors # @@ -984,6 +986,11 @@ def uid_sort(sort_keys, search_keys, charset) # end # } # + # Response handlers can also be added when the client is created before the + # receiver thread is started, by the +response_handlers+ argument to ::new. + # This ensures every server response is handled, including the #greeting. + # + # Related: #remove_response_handler, #response_handlers def add_response_handler(handler = nil, &block) raise ArgumentError, "two Procs are passed" if handler && block @response_handlers.push(block || handler) @@ -1099,6 +1106,12 @@ def idle_done # OpenSSL::SSL::SSLContext#set_params as parameters. # open_timeout:: Seconds to wait until a connection is opened # idle_response_timeout:: Seconds to wait until an IDLE response is received + # response_handlers:: A list of response handlers to be added before the + # receiver thread is started. This ensures every server + # response is handled, including the #greeting. Note + # that the greeting is handled in the current thread, + # but all other responses are handled in the receiver + # thread. # # The most common errors are: # @@ -1141,6 +1154,7 @@ def initialize(host, port_or_options = {}, @responses = Hash.new([].freeze) @tagged_responses = {} @response_handlers = [] + options[:response_handlers]&.each do |h| add_response_handler(h) end @tagged_response_arrival = new_cond @continued_command_tag = nil @continuation_request_arrival = new_cond @@ -1157,6 +1171,7 @@ def initialize(host, port_or_options = {}, if @greeting.name == "BYE" raise ByeResponseError, @greeting end + @response_handlers.each do |handler| handler.call(@greeting) end @client_thread = Thread.current @receiver_thread = Thread.start { diff --git a/test/net/imap/test_imap_response_handlers.rb b/test/net/imap/test_imap_response_handlers.rb index 900c2ebca..f513d867f 100644 --- a/test/net/imap/test_imap_response_handlers.rb +++ b/test/net/imap/test_imap_response_handlers.rb @@ -50,4 +50,41 @@ def teardown end end + test "::new with response_handlers kwarg" do + greeting = nil + expunges = [] + alerts = [] + untagged = 0 + handler0 = ->{ greeting ||= _1 } + handler1 = ->{ alerts << _1.data.text if _1 in {data: {code: {name: "ALERT"}}} } + handler2 = ->{ expunges << _1.data if _1 in {name: "EXPUNGE"} } + handler3 = ->{ untagged += 1 if _1.is_a?(Net::IMAP::UntaggedResponse) } + response_handlers = [handler0, handler1, handler2, handler3] + + run_fake_server_in_thread do |server| + port = server.port + imap = Net::IMAP.new("localhost", port:, response_handlers:) + assert_equal response_handlers, imap.response_handlers + refute_same response_handlers, imap.response_handlers + + # handler0 recieved the greeting and handler3 counted it + assert_equal imap.greeting, greeting + assert_equal 1, untagged + + server.on("NOOP") do |resp| + resp.untagged "1 EXPUNGE" + resp.untagged "1 EXPUNGE" + resp.untagged "OK [ALERT] The first alert." + resp.done_ok "[ALERT] Did you see the alert?" + end + + imap.noop + assert_equal 4, untagged + assert_equal [1, 1], expunges # from handler2 + assert_equal ["The first alert.", "Did you see the alert?"], alerts + ensure + imap&.logout! unless imap&.disconnected? + end + end + end From 7a200056848795874dab476e4f6b36137d453224 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sun, 20 Apr 2025 14:27:54 -0400 Subject: [PATCH 14/23] =?UTF-8?q?=E2=9C=85=20Fix=20backport=20for=20net-im?= =?UTF-8?q?ap=20v0.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Net::IMAP::Config` was introduced by `net-imap` v0.4. --- test/net/imap/test_response_reader.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb index 319dece22..28bd8571e 100644 --- a/test/net/imap/test_response_reader.rb +++ b/test/net/imap/test_response_reader.rb @@ -5,12 +5,7 @@ require "test/unit" class ResponseReaderTest < Test::Unit::TestCase - def setup - Net::IMAP.config.reset - end - class FakeClient - def config; @config ||= Net::IMAP.config.new end end def literal(str) "{#{str.bytesize}}\r\n#{str}" end From b87b2883f61354da88b2f863d630e7072c5641de Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 16 Apr 2025 23:11:24 -0400 Subject: [PATCH 15/23] =?UTF-8?q?=E2=9C=85=20Fix=20ruby=202.7=20compatabil?= =?UTF-8?q?ity=20in=20backported=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/net/imap/test_imap_response_handlers.rb | 30 +++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/test/net/imap/test_imap_response_handlers.rb b/test/net/imap/test_imap_response_handlers.rb index f513d867f..7e79cf6a6 100644 --- a/test/net/imap/test_imap_response_handlers.rb +++ b/test/net/imap/test_imap_response_handlers.rb @@ -37,16 +37,17 @@ def teardown assert_equal 2, imap.response_handlers.length imap.noop - assert_pattern do - responses => [ - [:block, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 1]], - [:proc, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 1]], - [:block, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 2]], - [:proc, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 2]], - [:block, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 3]], - [:proc, Net::IMAP::UntaggedResponse[name: "EXPUNGE", data: 3]], - ] - end + responses = responses[0, 6].map {|which, resp| + [which, resp.class, resp.name, resp.data] + } + assert_equal [ + [:block, Net::IMAP::UntaggedResponse, "EXPUNGE", 1], + [:proc, Net::IMAP::UntaggedResponse, "EXPUNGE", 1], + [:block, Net::IMAP::UntaggedResponse, "EXPUNGE", 2], + [:proc, Net::IMAP::UntaggedResponse, "EXPUNGE", 2], + [:block, Net::IMAP::UntaggedResponse, "EXPUNGE", 3], + [:proc, Net::IMAP::UntaggedResponse, "EXPUNGE", 3], + ], responses end end @@ -56,14 +57,15 @@ def teardown alerts = [] untagged = 0 handler0 = ->{ greeting ||= _1 } - handler1 = ->{ alerts << _1.data.text if _1 in {data: {code: {name: "ALERT"}}} } - handler2 = ->{ expunges << _1.data if _1 in {name: "EXPUNGE"} } - handler3 = ->{ untagged += 1 if _1.is_a?(Net::IMAP::UntaggedResponse) } + handler1 = ->(r) { alerts << r.data.text if r.data.code.name == "ALERT" rescue nil } + handler2 = ->(r) { expunges << r.data if r.name == "EXPUNGE" } + handler3 = ->(r) { untagged += 1 if r.is_a?(Net::IMAP::UntaggedResponse) } response_handlers = [handler0, handler1, handler2, handler3] run_fake_server_in_thread do |server| port = server.port - imap = Net::IMAP.new("localhost", port:, response_handlers:) + imap = Net::IMAP.new("localhost", port: port, + response_handlers: response_handlers) assert_equal response_handlers, imap.response_handlers refute_same response_handlers, imap.response_handlers From 64c0b51e9cc17ddab42c5fd6f19b1ca916322ca5 Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 14 Apr 2025 09:23:58 -0400 Subject: [PATCH 16/23] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Save=20ResponseReade?= =?UTF-8?q?r=20ivars:=20@buff=20&=20@literal=5Fsize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids the need to pass these to every method that uses them. That's not a big deal now, but it simplifies the next few changes. Also added a missing test for empty literals: "{0}\r\n" --- lib/net/imap/response_reader.rb | 20 ++++++++++++++------ test/net/imap/test_response_reader.rb | 3 +++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/net/imap/response_reader.rb b/lib/net/imap/response_reader.rb index 57770e3b8..7d3a53685 100644 --- a/lib/net/imap/response_reader.rb +++ b/lib/net/imap/response_reader.rb @@ -11,26 +11,34 @@ def initialize(client, sock) end def read_response_buffer - buff = String.new + @buff = String.new catch :eof do while true - read_line(buff) - break unless /\{(\d+)\}\r\n\z/n =~ buff - read_literal(buff, $1.to_i) + read_line + break unless (@literal_size = get_literal_size) + read_literal end end buff + ensure + @buff = nil end private - def read_line(buff) + attr_reader :buff, :literal_size + + def get_literal_size = /\{(\d+)\}\r\n\z/n =~ buff && $1.to_i + + def read_line buff << (@sock.gets(CRLF) or throw :eof) end - def read_literal(buff, literal_size) + def read_literal literal = String.new(capacity: literal_size) buff << (@sock.read(literal_size, literal) or throw :eof) + ensure + @literal_size = nil end end diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb index 28bd8571e..9a8c63dcd 100644 --- a/test/net/imap/test_response_reader.rb +++ b/test/net/imap/test_response_reader.rb @@ -19,6 +19,7 @@ def literal(str) "{#{str.bytesize}}\r\n#{str}" end long_line = "tag ok #{aaaaaaaaa} #{aaaaaaaaa}\r\n" literal_aaaa = "* fake #{literal aaaaaaaaa}\r\n" literal_crlf = "tag ok #{literal many_crlfs} #{literal many_crlfs}\r\n" + zero_literal = "tag ok #{literal ""} #{literal ""}\r\n" illegal_crs = "tag ok #{many_crs} #{many_crs}\r\n" illegal_lfs = "tag ok #{literal "\r"}\n#{literal "\r"}\n\r\n" io = StringIO.new([ @@ -26,6 +27,7 @@ def literal(str) "{#{str.bytesize}}\r\n#{str}" end long_line, literal_aaaa, literal_crlf, + zero_literal, illegal_crs, illegal_lfs, simple, @@ -35,6 +37,7 @@ def literal(str) "{#{str.bytesize}}\r\n#{str}" end assert_equal long_line, rcvr.read_response_buffer.to_str assert_equal literal_aaaa, rcvr.read_response_buffer.to_str assert_equal literal_crlf, rcvr.read_response_buffer.to_str + assert_equal zero_literal, rcvr.read_response_buffer.to_str assert_equal illegal_crs, rcvr.read_response_buffer.to_str assert_equal illegal_lfs, rcvr.read_response_buffer.to_str assert_equal simple, rcvr.read_response_buffer.to_str From 15b6a65e655da6e2f7240e30fbaddd7a15b3b7a1 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sat, 22 Mar 2025 15:33:02 -0400 Subject: [PATCH 17/23] =?UTF-8?q?=E2=9C=85=20Fix=20backport=20to=20net-ima?= =?UTF-8?q?p=200.3=20and=20ruby=202.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FakeServer was introduced by v0.4, so the tests needed to be rewritten without it. And ruby 2.6 doesn't support numbered params or "...". --- test/net/imap/test_imap_response_handlers.rb | 86 +++++++++++++++----- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/test/net/imap/test_imap_response_handlers.rb b/test/net/imap/test_imap_response_handlers.rb index 7e79cf6a6..3786f2427 100644 --- a/test/net/imap/test_imap_response_handlers.rb +++ b/test/net/imap/test_imap_response_handlers.rb @@ -2,13 +2,10 @@ require "net/imap" require "test/unit" -require_relative "fake_server" class IMAPResponseHandlersTest < Test::Unit::TestCase - include Net::IMAP::FakeServer::TestHelper def setup - Net::IMAP.config.reset @do_not_reverse_lookup = Socket.do_not_reverse_lookup Socket.do_not_reverse_lookup = true @threads = [] @@ -23,17 +20,32 @@ def teardown end test "#add_response_handlers" do - responses = [] - with_fake_server do |server, imap| - server.on("NOOP") do |resp| - 3.times do resp.untagged("#{_1 + 1} EXPUNGE") end - resp.done_ok + server = create_tcp_server + port = server.addr[1] + start_server do + sock = server.accept + Timeout.timeout(5) do + sock.print("* OK connection established\r\n") + sock.gets # => NOOP + sock.print("* 1 EXPUNGE\r\n") + sock.print("* 2 EXPUNGE\r\n") + sock.print("* 3 EXPUNGE\r\n") + sock.print("RUBY0001 OK NOOP completed\r\n") + sock.gets # => LOGOUT + sock.print("* BYE terminating connection\r\n") + sock.print("RUBY0002 OK LOGOUT completed\r\n") + ensure + sock.close + server.close end - + end + begin + responses = [] + imap = Net::IMAP.new(server_addr, port: port) assert_equal 0, imap.response_handlers.length - imap.add_response_handler do responses << [:block, _1] end + imap.add_response_handler do |r| responses << [:block, r] end assert_equal 1, imap.response_handlers.length - imap.add_response_handler(->{ responses << [:proc, _1] }) + imap.add_response_handler(->(r) { responses << [:proc, r] }) assert_equal 2, imap.response_handlers.length imap.noop @@ -48,6 +60,9 @@ def teardown [:block, Net::IMAP::UntaggedResponse, "EXPUNGE", 3], [:proc, Net::IMAP::UntaggedResponse, "EXPUNGE", 3], ], responses + ensure + imap&.logout + imap&.disconnect end end @@ -56,14 +71,32 @@ def teardown expunges = [] alerts = [] untagged = 0 - handler0 = ->{ greeting ||= _1 } + handler0 = ->(r) { greeting ||= r } handler1 = ->(r) { alerts << r.data.text if r.data.code.name == "ALERT" rescue nil } handler2 = ->(r) { expunges << r.data if r.name == "EXPUNGE" } handler3 = ->(r) { untagged += 1 if r.is_a?(Net::IMAP::UntaggedResponse) } response_handlers = [handler0, handler1, handler2, handler3] - run_fake_server_in_thread do |server| - port = server.port + server = create_tcp_server + port = server.addr[1] + start_server do + sock = server.accept + Timeout.timeout(5) do + sock.print("* OK connection established\r\n") + sock.gets # => NOOP + sock.print("* 1 EXPUNGE\r\n") + sock.print("* 1 EXPUNGE\r\n") + sock.print("* OK [ALERT] The first alert.\r\n") + sock.print("RUBY0001 OK [ALERT] Did you see the alert?\r\n") + sock.gets # => LOGOUT + sock.print("* BYE terminating connection\r\n") + sock.print("RUBY0002 OK LOGOUT completed\r\n") + ensure + sock.close + server.close + end + end + begin imap = Net::IMAP.new("localhost", port: port, response_handlers: response_handlers) assert_equal response_handlers, imap.response_handlers @@ -73,20 +106,29 @@ def teardown assert_equal imap.greeting, greeting assert_equal 1, untagged - server.on("NOOP") do |resp| - resp.untagged "1 EXPUNGE" - resp.untagged "1 EXPUNGE" - resp.untagged "OK [ALERT] The first alert." - resp.done_ok "[ALERT] Did you see the alert?" - end - imap.noop assert_equal 4, untagged assert_equal [1, 1], expunges # from handler2 assert_equal ["The first alert.", "Did you see the alert?"], alerts ensure - imap&.logout! unless imap&.disconnected? + imap&.logout + imap&.disconnect end end + def start_server + th = Thread.new do + yield + end + @threads << th + sleep 0.1 until th.stop? + end + + def create_tcp_server + return TCPServer.new(server_addr, 0) + end + + def server_addr + Addrinfo.tcp("localhost", 0).ip_address + end end From 335872f4620132673ff0b096a19b574d66f13210 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sat, 19 Apr 2025 21:30:58 -0400 Subject: [PATCH 18/23] =?UTF-8?q?=E2=9C=85=20Fix=20backport=20compatibilit?= =?UTF-8?q?y=20with=20ruby=202.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/response_reader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/net/imap/response_reader.rb b/lib/net/imap/response_reader.rb index 7d3a53685..6a608b163 100644 --- a/lib/net/imap/response_reader.rb +++ b/lib/net/imap/response_reader.rb @@ -28,7 +28,7 @@ def read_response_buffer attr_reader :buff, :literal_size - def get_literal_size = /\{(\d+)\}\r\n\z/n =~ buff && $1.to_i + def get_literal_size; /\{(\d+)\}\r\n\z/n =~ buff && $1.to_i end def read_line buff << (@sock.gets(CRLF) or throw :eof) From 20c16a2eaec1dc6775675abbd8f3f2c412e7533f Mon Sep 17 00:00:00 2001 From: nick evans Date: Sat, 19 Apr 2025 22:21:59 -0400 Subject: [PATCH 19/23] =?UTF-8?q?=E2=9C=A8=20Limit=20max=20response=20size?= =?UTF-8?q?=20to=20512MiB=20(hard-coded)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _Please note:_ this only limits the size per response. It does _not_ limit how many unhandled responses may be stored on the responses hash. --- lib/net/imap/errors.rb | 33 ++++++++++++++++++++++ lib/net/imap/response_reader.rb | 31 +++++++++++++++++++-- test/net/imap/test_errors.rb | 40 +++++++++++++++++++++++++++ test/net/imap/test_response_reader.rb | 23 +++++++++++++++ 4 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 test/net/imap/test_errors.rb diff --git a/lib/net/imap/errors.rb b/lib/net/imap/errors.rb index 33dd541ce..9f7cf92da 100644 --- a/lib/net/imap/errors.rb +++ b/lib/net/imap/errors.rb @@ -11,6 +11,39 @@ class Error < StandardError class DataFormatError < Error end + # Error raised when the socket cannot be read, due to a configured limit. + class ResponseReadError < Error + end + + # Error raised when a response is larger than IMAP#max_response_size. + class ResponseTooLargeError < ResponseReadError + attr_reader :bytes_read, :literal_size + attr_reader :max_response_size + + def initialize(msg = nil, *args, + bytes_read: nil, + literal_size: nil, + max_response_size: nil, + **kwargs) + @bytes_read = bytes_read + @literal_size = literal_size + @max_response_size = max_response_size + msg ||= [ + "Response size", response_size_msg, "exceeds max_response_size", + max_response_size && "(#{max_response_size}B)", + ].compact.join(" ") + super(msg, *args, **kwargs) + end + + private + + def response_size_msg + if bytes_read && literal_size + "(#{bytes_read}B read + #{literal_size}B literal)" + end + end + end + # Error raised when a response from the server is non-parseable. class ResponseParseError < Error end diff --git a/lib/net/imap/response_reader.rb b/lib/net/imap/response_reader.rb index 6a608b163..3c33dea39 100644 --- a/lib/net/imap/response_reader.rb +++ b/lib/net/imap/response_reader.rb @@ -28,19 +28,46 @@ def read_response_buffer attr_reader :buff, :literal_size + def bytes_read = buff.bytesize + def empty? = buff.empty? + def done? = line_done? && !get_literal_size + def line_done? = buff.end_with?(CRLF) def get_literal_size; /\{(\d+)\}\r\n\z/n =~ buff && $1.to_i end def read_line - buff << (@sock.gets(CRLF) or throw :eof) + buff << (@sock.gets(CRLF, read_limit) or throw :eof) + max_response_remaining! unless line_done? end def read_literal + # check before allocating memory for literal + max_response_remaining! literal = String.new(capacity: literal_size) - buff << (@sock.read(literal_size, literal) or throw :eof) + buff << (@sock.read(read_limit(literal_size), literal) or throw :eof) ensure @literal_size = nil end + def read_limit(limit = nil) + [limit, max_response_remaining!].compact.min + end + + def max_response_size = 512 << 20 # TODO: Config#max_response_size + def max_response_remaining = max_response_size &.- bytes_read + def response_too_large? = max_response_size &.< min_response_size + def min_response_size = bytes_read + min_response_remaining + + def min_response_remaining + empty? ? 3 : done? ? 0 : (literal_size || 0) + 2 + end + + def max_response_remaining! + return max_response_remaining unless response_too_large? + raise ResponseTooLargeError.new( + max_response_size:, bytes_read:, literal_size:, + ) + end + end end end diff --git a/test/net/imap/test_errors.rb b/test/net/imap/test_errors.rb new file mode 100644 index 000000000..a6a7cb0f4 --- /dev/null +++ b/test/net/imap/test_errors.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require "net/imap" +require "test/unit" + +class IMAPErrorsTest < Test::Unit::TestCase + + test "ResponseTooLargeError" do + err = Net::IMAP::ResponseTooLargeError.new + assert_nil err.bytes_read + assert_nil err.literal_size + assert_nil err.max_response_size + + err = Net::IMAP::ResponseTooLargeError.new("manually set message") + assert_equal "manually set message", err.message + assert_nil err.bytes_read + assert_nil err.literal_size + assert_nil err.max_response_size + + err = Net::IMAP::ResponseTooLargeError.new(max_response_size: 1024) + assert_equal "Response size exceeds max_response_size (1024B)", err.message + assert_nil err.bytes_read + assert_nil err.literal_size + assert_equal 1024, err.max_response_size + + err = Net::IMAP::ResponseTooLargeError.new(bytes_read: 1200, + max_response_size: 1200) + assert_equal 1200, err.bytes_read + assert_equal "Response size exceeds max_response_size (1200B)", err.message + + err = Net::IMAP::ResponseTooLargeError.new(bytes_read: 800, + literal_size: 1000, + max_response_size: 1200) + assert_equal 800, err.bytes_read + assert_equal 1000, err.literal_size + assert_equal("Response size (800B read + 1000B literal) " \ + "exceeds max_response_size (1200B)", err.message) + end + +end diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb index 9a8c63dcd..5a1491bee 100644 --- a/test/net/imap/test_response_reader.rb +++ b/test/net/imap/test_response_reader.rb @@ -44,4 +44,27 @@ def literal(str) "{#{str.bytesize}}\r\n#{str}" end assert_equal "", rcvr.read_response_buffer.to_str end + class LimitedResponseReader < Net::IMAP::ResponseReader + attr_reader :max_response_size + def initialize(*args, max_response_size:) + super(*args) + @max_response_size = max_response_size + end + end + + test "#read_response_buffer with max_response_size" do + client = FakeClient.new + max_response_size = 10 + under = "+ 3456\r\n" + exact = "+ 345678\r\n" + over = "+ 3456789\r\n" + io = StringIO.new([under, exact, over].join) + rcvr = LimitedResponseReader.new(client, io, max_response_size:) + assert_equal under, rcvr.read_response_buffer.to_str + assert_equal exact, rcvr.read_response_buffer.to_str + assert_raise Net::IMAP::ResponseTooLargeError do + rcvr.read_response_buffer + end + end + end From 5431e16b779254ad7b2786e4367bc04328418264 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sun, 20 Apr 2025 17:38:43 -0400 Subject: [PATCH 20/23] =?UTF-8?q?=E2=9C=A8=20Make=20max=5Fresponse=5Fsize?= =?UTF-8?q?=20configurable=20[=F0=9F=9A=A7=20partial]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note that this cherry-picked commit is missing key paits that are incompatible with net-imap before 0.4. I'm keeping the conflict resolution here, and the updates for net-imap 0.3 in the next commit. ------ Though it would be useful to also have limits based on response type and what commands are currently running, that's out of scope for now. _Please note:_ this only limits the size per response. It does _not_ limit how many unhandled responses may be stored on the responses hash. --- lib/net/imap.rb | 15 +++++ lib/net/imap/response_reader.rb | 2 +- test/net/imap/test_imap_max_response_size.rb | 67 ++++++++++++++++++++ test/net/imap/test_response_reader.rb | 13 +--- 4 files changed, 86 insertions(+), 11 deletions(-) create mode 100644 test/net/imap/test_imap_max_response_size.rb diff --git a/lib/net/imap.rb b/lib/net/imap.rb index f00491008..4f0bb7c6a 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -133,6 +133,10 @@ module Net # # Use paginated or limited versions of commands whenever possible. # + # Use Config#max_response_size to impose a limit on incoming server responses + # as they are being read. This is especially important for untrusted + # servers. + # # Use #add_response_handler to handle responses after each one is received. # Use the +response_handlers+ argument to ::new to assign response handlers # before the receiver thread is started. @@ -313,6 +317,17 @@ class << self alias default_ssl_port default_tls_port end + ## + # :attr_accessor: max_response_size + # + # The maximum allowed server response size, in bytes. + # Delegates to {config.max_response_size}[rdoc-ref:Config#max_response_size]. + + # :stopdoc: + def max_response_size; config.max_response_size end + def max_response_size=(val) config.max_response_size = val end + # :startdoc: + # Disconnects from the server. def disconnect return if disconnected? diff --git a/lib/net/imap/response_reader.rb b/lib/net/imap/response_reader.rb index 3c33dea39..bcf360172 100644 --- a/lib/net/imap/response_reader.rb +++ b/lib/net/imap/response_reader.rb @@ -52,7 +52,7 @@ def read_limit(limit = nil) [limit, max_response_remaining!].compact.min end - def max_response_size = 512 << 20 # TODO: Config#max_response_size + def max_response_size = client.max_response_size def max_response_remaining = max_response_size &.- bytes_read def response_too_large? = max_response_size &.< min_response_size def min_response_size = bytes_read + min_response_remaining diff --git a/test/net/imap/test_imap_max_response_size.rb b/test/net/imap/test_imap_max_response_size.rb new file mode 100644 index 000000000..3751d0bc8 --- /dev/null +++ b/test/net/imap/test_imap_max_response_size.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require "net/imap" +require "test/unit" +require_relative "fake_server" + +class IMAPMaxResponseSizeTest < Test::Unit::TestCase + include Net::IMAP::FakeServer::TestHelper + + def setup + Net::IMAP.config.reset + @do_not_reverse_lookup = Socket.do_not_reverse_lookup + Socket.do_not_reverse_lookup = true + @threads = [] + end + + def teardown + if !@threads.empty? + assert_join_threads(@threads) + end + ensure + Socket.do_not_reverse_lookup = @do_not_reverse_lookup + end + + test "#max_response_size reading literals" do + with_fake_server(preauth: true) do |server, imap| + imap.max_response_size = 12_345 + 30 + server.on("NOOP") do |resp| + resp.untagged("1 FETCH (BODY[] {12345}\r\n" + "a" * 12_345 + ")") + resp.done_ok + end + imap.noop + assert_equal "a" * 12_345, imap.responses("FETCH").first.message + end + end + + test "#max_response_size closes connection for too long line" do + Net::IMAP.config.max_response_size = 10 + run_fake_server_in_thread(preauth: false, ignore_io_error: true) do |server| + assert_raise_with_message( + Net::IMAP::ResponseTooLargeError, /exceeds max_response_size .*\b10B\b/ + ) do + with_client("localhost", port: server.port) do + fail "should not get here (greeting longer than max_response_size)" + end + end + end + end + + test "#max_response_size closes connection for too long literal" do + Net::IMAP.config.max_response_size = 1<<20 + with_fake_server(preauth: false, ignore_io_error: true) do |server, client| + client.max_response_size = 50 + server.on("NOOP") do |resp| + resp.untagged("1 FETCH (BODY[] {1000}\r\n" + "a" * 1000 + ")") + end + assert_raise_with_message( + Net::IMAP::ResponseTooLargeError, + /\d+B read \+ 1000B literal.* exceeds max_response_size .*\b50B\b/ + ) do + client.noop + fail "should not get here (FETCH literal longer than max_response_size)" + end + end + end + +end diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb index 5a1491bee..6b58d555b 100644 --- a/test/net/imap/test_response_reader.rb +++ b/test/net/imap/test_response_reader.rb @@ -6,6 +6,7 @@ class ResponseReaderTest < Test::Unit::TestCase class FakeClient + def max_response_size = config.max_response_size end def literal(str) "{#{str.bytesize}}\r\n#{str}" end @@ -44,22 +45,14 @@ def literal(str) "{#{str.bytesize}}\r\n#{str}" end assert_equal "", rcvr.read_response_buffer.to_str end - class LimitedResponseReader < Net::IMAP::ResponseReader - attr_reader :max_response_size - def initialize(*args, max_response_size:) - super(*args) - @max_response_size = max_response_size - end - end - test "#read_response_buffer with max_response_size" do client = FakeClient.new - max_response_size = 10 + client.config.max_response_size = 10 under = "+ 3456\r\n" exact = "+ 345678\r\n" over = "+ 3456789\r\n" io = StringIO.new([under, exact, over].join) - rcvr = LimitedResponseReader.new(client, io, max_response_size:) + rcvr = Net::IMAP::ResponseReader.new(client, io) assert_equal under, rcvr.read_response_buffer.to_str assert_equal exact, rcvr.read_response_buffer.to_str assert_raise Net::IMAP::ResponseTooLargeError do From 450bb4d757d9b9f2866ebd6e1efdd5d94a311b05 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sun, 20 Apr 2025 21:01:48 -0400 Subject: [PATCH 21/23] =?UTF-8?q?=E2=9C=85=20Fix=20backport=20compatibilit?= =?UTF-8?q?y=20with=20ruby=202.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/response_reader.rb | 20 +++++++++++--------- test/net/imap/test_response_reader.rb | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/net/imap/response_reader.rb b/lib/net/imap/response_reader.rb index bcf360172..fd7561fa7 100644 --- a/lib/net/imap/response_reader.rb +++ b/lib/net/imap/response_reader.rb @@ -28,10 +28,10 @@ def read_response_buffer attr_reader :buff, :literal_size - def bytes_read = buff.bytesize - def empty? = buff.empty? - def done? = line_done? && !get_literal_size - def line_done? = buff.end_with?(CRLF) + def bytes_read; buff.bytesize end + def empty?; buff.empty? end + def done?; line_done? && !get_literal_size end + def line_done?; buff.end_with?(CRLF) end def get_literal_size; /\{(\d+)\}\r\n\z/n =~ buff && $1.to_i end def read_line @@ -52,10 +52,10 @@ def read_limit(limit = nil) [limit, max_response_remaining!].compact.min end - def max_response_size = client.max_response_size - def max_response_remaining = max_response_size &.- bytes_read - def response_too_large? = max_response_size &.< min_response_size - def min_response_size = bytes_read + min_response_remaining + def max_response_size; client.max_response_size end + def max_response_remaining; max_response_size &.- bytes_read end + def response_too_large?; max_response_size &.< min_response_size end + def min_response_size; bytes_read + min_response_remaining end def min_response_remaining empty? ? 3 : done? ? 0 : (literal_size || 0) + 2 @@ -64,7 +64,9 @@ def min_response_remaining def max_response_remaining! return max_response_remaining unless response_too_large? raise ResponseTooLargeError.new( - max_response_size:, bytes_read:, literal_size:, + max_response_size: max_response_size, + bytes_read: bytes_read, + literal_size: literal_size, ) end diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb index 6b58d555b..716922d99 100644 --- a/test/net/imap/test_response_reader.rb +++ b/test/net/imap/test_response_reader.rb @@ -6,7 +6,7 @@ class ResponseReaderTest < Test::Unit::TestCase class FakeClient - def max_response_size = config.max_response_size + def max_response_size; config.max_response_size end end def literal(str) "{#{str.bytesize}}\r\n#{str}" end From 673cab874374670fca850dc0e16ddc62ee3b8a68 Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 9 Apr 2025 09:54:51 -0400 Subject: [PATCH 22/23] =?UTF-8?q?=E2=9C=85=20Fix=20backport=20to=20not-ima?= =?UTF-8?q?p=200.3=20and=20ruby=202.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For the net-imap v0.3 backport, two major changes were needed: * the tests needed to be almost completely rewritten because FakeServer was added for v0.4. * `max_response_size` needed to be on Net::IMAP directly, because Config was added for v0.4. --- lib/net/imap.rb | 49 +++++++-- lib/net/imap/errors.rb | 1 + test/net/imap/test_imap_max_response_size.rb | 109 +++++++++++++------ test/net/imap/test_response_reader.rb | 4 +- 4 files changed, 118 insertions(+), 45 deletions(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 4f0bb7c6a..855b41a01 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -133,7 +133,7 @@ module Net # # Use paginated or limited versions of commands whenever possible. # - # Use Config#max_response_size to impose a limit on incoming server responses + # Use #max_response_size to impose a limit on incoming server responses # as they are being read. This is especially important for untrusted # servers. # @@ -288,6 +288,40 @@ class IMAP < Protocol # Seconds to wait until an IDLE response is received. attr_reader :idle_response_timeout + # The maximum allowed server response size. When +nil+, there is no limit + # on response size. + # + # The default value is _unlimited_ (after +v0.5.8+, the default is 512 MiB). + # A _much_ lower value should be used with untrusted servers (for example, + # when connecting to a user-provided hostname). When using a lower limit, + # message bodies should be fetched in chunks rather than all at once. + # + # Please Note: this only limits the size per response. It does + # not prevent a flood of individual responses and it does not limit how + # many unhandled responses may be stored on the responses hash. See + # Net::IMAP@Unbounded+memory+use. + # + # Socket reads are limited to the maximum remaining bytes for the current + # response: max_response_size minus the bytes that have already been read. + # When the limit is reached, or reading a +literal+ _would_ go over the + # limit, ResponseTooLargeError is raised and the connection is closed. + # See also #socket_read_limit. + # + # Note that changes will not take effect immediately, because the receiver + # thread may already be waiting for the next response using the previous + # value. Net::IMAP#noop can force a response and enforce the new setting + # immediately. + # + # ==== Versioned Defaults + # + # Net::IMAP#max_response_size was added in +v0.2.5+ and +v0.3.9+ as an + # attr_accessor, and in +v0.4.20+ and +v0.5.7+ as a delegator to a config + # attribute. + # + # * original: +nil+ (no limit) + # * +0.5+: 512 MiB + attr_accessor :max_response_size + # The thread to receive exceptions. attr_accessor :client_thread @@ -317,17 +351,6 @@ class << self alias default_ssl_port default_tls_port end - ## - # :attr_accessor: max_response_size - # - # The maximum allowed server response size, in bytes. - # Delegates to {config.max_response_size}[rdoc-ref:Config#max_response_size]. - - # :stopdoc: - def max_response_size; config.max_response_size end - def max_response_size=(val) config.max_response_size = val end - # :startdoc: - # Disconnects from the server. def disconnect return if disconnected? @@ -1129,6 +1152,7 @@ def idle_done # that the greeting is handled in the current thread, # but all other responses are handled in the receiver # thread. + # max_response_size:: See #max_response_size. # # The most common errors are: # @@ -1159,6 +1183,7 @@ def initialize(host, port_or_options = {}, @tagno = 0 @open_timeout = options[:open_timeout] || 30 @idle_response_timeout = options[:idle_response_timeout] || 5 + @max_response_size = options[:max_response_size] @parser = ResponseParser.new @sock = tcp_socket(@host, @port) @reader = ResponseReader.new(self, @sock) diff --git a/lib/net/imap/errors.rb b/lib/net/imap/errors.rb index 9f7cf92da..d53bc7a10 100644 --- a/lib/net/imap/errors.rb +++ b/lib/net/imap/errors.rb @@ -32,6 +32,7 @@ def initialize(msg = nil, *args, "Response size", response_size_msg, "exceeds max_response_size", max_response_size && "(#{max_response_size}B)", ].compact.join(" ") + return super(msg, *args) if kwargs.empty? # ruby 2.6 compatibility super(msg, *args, **kwargs) end diff --git a/test/net/imap/test_imap_max_response_size.rb b/test/net/imap/test_imap_max_response_size.rb index 3751d0bc8..7ec554c3b 100644 --- a/test/net/imap/test_imap_max_response_size.rb +++ b/test/net/imap/test_imap_max_response_size.rb @@ -2,13 +2,10 @@ require "net/imap" require "test/unit" -require_relative "fake_server" class IMAPMaxResponseSizeTest < Test::Unit::TestCase - include Net::IMAP::FakeServer::TestHelper def setup - Net::IMAP.config.reset @do_not_reverse_lookup = Socket.do_not_reverse_lookup Socket.do_not_reverse_lookup = true @threads = [] @@ -23,45 +20,95 @@ def teardown end test "#max_response_size reading literals" do - with_fake_server(preauth: true) do |server, imap| + _, port = with_server_socket do |sock| + sock.gets # => NOOP + sock.print("RUBY0001 OK done\r\n") + sock.gets # => NOOP + sock.print("* 1 FETCH (BODY[] {12345}\r\n" + "a" * 12_345 + ")\r\n") + sock.print("RUBY0002 OK done\r\n") + "RUBY0003" + end + Timeout.timeout(5) do + imap = Net::IMAP.new("localhost", port: port, max_response_size: 640 << 20) + assert_equal 640 << 20, imap.max_response_size imap.max_response_size = 12_345 + 30 - server.on("NOOP") do |resp| - resp.untagged("1 FETCH (BODY[] {12345}\r\n" + "a" * 12_345 + ")") - resp.done_ok - end - imap.noop - assert_equal "a" * 12_345, imap.responses("FETCH").first.message + assert_equal 12_345 + 30, imap.max_response_size + imap.noop # to reset the get_response limit + imap.noop # to send the FETCH + assert_equal "a" * 12_345, imap.responses["FETCH"].first.attr["BODY[]"] + ensure + imap.logout rescue nil + imap.disconnect rescue nil end end test "#max_response_size closes connection for too long line" do - Net::IMAP.config.max_response_size = 10 - run_fake_server_in_thread(preauth: false, ignore_io_error: true) do |server| - assert_raise_with_message( - Net::IMAP::ResponseTooLargeError, /exceeds max_response_size .*\b10B\b/ - ) do - with_client("localhost", port: server.port) do - fail "should not get here (greeting longer than max_response_size)" - end - end + _, port = with_server_socket do |sock| + sock.gets or next # => never called + fail "client disconnects first" + end + assert_raise_with_message( + Net::IMAP::ResponseTooLargeError, /exceeds max_response_size .*\b10B\b/ + ) do + Net::IMAP.new("localhost", port: port, max_response_size: 10) + fail "should not get here (greeting longer than max_response_size)" end end test "#max_response_size closes connection for too long literal" do - Net::IMAP.config.max_response_size = 1<<20 - with_fake_server(preauth: false, ignore_io_error: true) do |server, client| - client.max_response_size = 50 - server.on("NOOP") do |resp| - resp.untagged("1 FETCH (BODY[] {1000}\r\n" + "a" * 1000 + ")") - end - assert_raise_with_message( - Net::IMAP::ResponseTooLargeError, - /\d+B read \+ 1000B literal.* exceeds max_response_size .*\b50B\b/ - ) do - client.noop - fail "should not get here (FETCH literal longer than max_response_size)" + _, port = with_server_socket(ignore_io_error: true) do |sock| + sock.gets # => NOOP + sock.print "* 1 FETCH (BODY[] {1000}\r\n" + "a" * 1000 + ")\r\n" + sock.print("RUBY0001 OK done\r\n") + end + client = Net::IMAP.new("localhost", port: port, max_response_size: 1000) + assert_equal 1000, client.max_response_size + client.max_response_size = 50 + assert_equal 50, client.max_response_size + assert_raise_with_message( + Net::IMAP::ResponseTooLargeError, + /\d+B read \+ 1000B literal.* exceeds max_response_size .*\b50B\b/ + ) do + client.noop + fail "should not get here (FETCH literal longer than max_response_size)" + end + end + + def with_server_socket(ignore_io_error: false) + server = create_tcp_server + port = server.addr[1] + start_server do + Timeout.timeout(5) do + sock = server.accept + sock.print("* OK connection established\r\n") + logout_tag = yield sock if block_given? + sock.gets # => LOGOUT + sock.print("* BYE terminating connection\r\n") + sock.print("#{logout_tag} OK LOGOUT completed\r\n") if logout_tag + rescue IOError, EOFError, Errno::ECONNABORTED, Errno::ECONNRESET, + Errno::EPIPE, Errno::ETIMEDOUT + ignore_io_error or raise + ensure + sock.close rescue nil + server.close rescue nil end end + return server, port + end + + def start_server + th = Thread.new do + yield + end + @threads << th + sleep 0.1 until th.stop? end + def create_tcp_server + return TCPServer.new(server_addr, 0) + end + + def server_addr + Addrinfo.tcp("localhost", 0).ip_address + end end diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb index 716922d99..d2c1c11aa 100644 --- a/test/net/imap/test_response_reader.rb +++ b/test/net/imap/test_response_reader.rb @@ -6,7 +6,7 @@ class ResponseReaderTest < Test::Unit::TestCase class FakeClient - def max_response_size; config.max_response_size end + attr_accessor :max_response_size end def literal(str) "{#{str.bytesize}}\r\n#{str}" end @@ -47,7 +47,7 @@ def literal(str) "{#{str.bytesize}}\r\n#{str}" end test "#read_response_buffer with max_response_size" do client = FakeClient.new - client.config.max_response_size = 10 + client.max_response_size = 10 under = "+ 3456\r\n" exact = "+ 345678\r\n" over = "+ 3456789\r\n" From c5a35d29ff9e77a4d0b184a7eaf90314cd4acb59 Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 21 Apr 2025 23:03:20 -0400 Subject: [PATCH 23/23] =?UTF-8?q?=F0=9F=94=96=20Bump=20version=20to=20v0.2?= =?UTF-8?q?.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 855b41a01..6d1ef845b 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -255,7 +255,7 @@ module Net # Unicode", RFC-2152[https://tools.ietf.org/html/rfc2152], May 1997. # class IMAP < Protocol - VERSION = "0.2.4" + VERSION = "0.2.5" autoload :ResponseReader, File.expand_path("imap/response_reader", __dir__)