From b7ed240a8fb7b6e9ccb06bd43dd9c8d4620c7a81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 11:43:40 -0500 Subject: [PATCH 01/34] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20step-security?= =?UTF-8?q?/harden-runner=20from=202.10.2=20to=202.10.3=20(#375)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [step-security/harden-runner](https://github.com/step-security/harden-runner) from 2.10.2 to 2.10.3. - [Release notes](https://github.com/step-security/harden-runner/releases) - [Commits](https://github.com/step-security/harden-runner/compare/0080882f6c36860b6ba35c610c98ce87d4e2f26f...c95a14d0e5bab51a9f56296a4eb0e416910cd350) --- updated-dependencies: - dependency-name: step-security/harden-runner dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/push_gem.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push_gem.yml b/.github/workflows/push_gem.yml index b5a7bbf9..e4a5f7ba 100644 --- a/.github/workflows/push_gem.yml +++ b/.github/workflows/push_gem.yml @@ -24,7 +24,7 @@ jobs: steps: # Set up - name: Harden Runner - uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: egress-policy: audit From 0eb43902f00f6c567aab7d98a67b719159268e4a Mon Sep 17 00:00:00 2001 From: nick evans Date: Thu, 16 Jan 2025 21:25:53 -0500 Subject: [PATCH 02/34] =?UTF-8?q?=F0=9F=93=9A=20Update=20SequenceSet=20nom?= =?UTF-8?q?alized=20vs=20ordered=20rdoc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/sequence_set.rb | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index ea8b0a67..1cb85112 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -56,18 +56,20 @@ class IMAP # set = Net::IMAP::SequenceSet[1, 2, [3..7, 5], 6..10, 2048, 1024] # set.valid_string #=> "1:10,55,1024:2048" # - # == Normalized form + # == Ordered and Normalized sets # - # When a sequence set is created with a single String value, that #string - # representation is preserved. SequenceSet's internal representation - # implicitly sorts all entries, de-duplicates numbers, and coalesces - # adjacent or overlapping ranges. Most enumeration methods and offset-based - # methods use this normalized representation. Most modification methods - # will convert #string to its normalized form. + # Sometimes the order of the set's members is significant, such as with the + # +ESORT+, CONTEXT=SORT, and +UIDPLUS+ extensions. So, when a + # sequence set is created by the parser or with a single string value, that + # #string representation is preserved. # - # In some cases the order of the string representation is significant, such - # as the +ESORT+, CONTEXT=SORT, and +UIDPLUS+ extensions. Use - # #entries or #each_entry to enumerate the set in its original order. To + # Internally, SequenceSet stores a normalized representation which sorts all + # entries, de-duplicates numbers, and coalesces adjacent or overlapping + # ranges. Most methods use this normalized representation to achieve + # O(lg n) porformance. Use #entries or #each_entry to enumerate + # the set in its original order. + # + # Most modification methods convert #string to its normalized form. To # preserve #string order while modifying a set, use #append, #string=, or # #replace. # @@ -181,7 +183,7 @@ class IMAP # - #max: Returns the maximum number in the set. # - #minmax: Returns the minimum and maximum numbers in the set. # - # Accessing value by offset: + # Accessing value by (normalized) offset: # - #[] (aliased as #slice): Returns the number or consecutive subset at a # given offset or range of offsets. # - #at: Returns the number at a given offset. @@ -189,6 +191,7 @@ class IMAP # # Set cardinality: # - #count (aliased as #size): Returns the count of numbers in the set. + # Duplicated numbers are not counted. # - #empty?: Returns whether the set has no members. \IMAP syntax does not # allow empty sequence sets. # - #valid?: Returns whether the set has any members. @@ -838,8 +841,8 @@ def entries; each_entry.to_a end # * translates to an endless range. Use #limit to translate both # cases to a maximum value. # - # If the original input was unordered or contains overlapping ranges, the - # returned ranges will be ordered and coalesced. + # The returned elements will be sorted and coalesced, even when the input + # #string is not. * will sort last. See #normalize. # # Net::IMAP::SequenceSet["2,5:9,6,*,12:11"].elements # #=> [2, 5..9, 11..12, :*] @@ -857,7 +860,7 @@ def elements; each_element.to_a end # translates to :*... Use #limit to set * to a maximum # value. # - # The returned ranges will be ordered and coalesced, even when the input + # The returned ranges will be sorted and coalesced, even when the input # #string is not. * will sort last. See #normalize. # # Net::IMAP::SequenceSet["2,5:9,6,*,12:11"].ranges From 4c9b1a27a48996a0a2f2bbe34a4433c87140f070 Mon Sep 17 00:00:00 2001 From: nick evans Date: Thu, 16 Jan 2025 21:53:02 -0500 Subject: [PATCH 03/34] =?UTF-8?q?=F0=9F=90=9B=20Fix=20`SequenceSet#append`?= =?UTF-8?q?=20when=20`@string`=20is=20nil?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/sequence_set.rb | 3 ++- test/net/imap/test_sequence_set.rb | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index 1cb85112..b046b0eb 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -684,8 +684,9 @@ def append(object) modifying! tuple = input_to_tuple object entry = tuple_to_str tuple + string unless empty? # write @string before tuple_add tuple_add tuple - @string = -(string ? "#{@string},#{entry}" : entry) + @string = -(@string ? "#{@string},#{entry}" : entry) self end diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index 85204b24..94c096ba 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -344,6 +344,14 @@ def obj.to_sequence_set; 192_168.001_255 end assert_equal "1:6,4:9", SequenceSet.new("1:6").append("4:9").string assert_equal "1:4,5:*", SequenceSet.new("1:4").append(5..).string assert_equal "5:*,1:4", SequenceSet.new("5:*").append(1..4).string + # also works from empty + assert_equal "5,1", SequenceSet.new.append(5).append(1).string + # also works when *previously* input was non-strings + assert_equal "*,1", SequenceSet.new(:*).append(1).string + assert_equal "1,5", SequenceSet.new(1).append("5").string + assert_equal "1:6,4:9", SequenceSet.new(1..6).append(4..9).string + assert_equal "1:4,5:*", SequenceSet.new(1..4).append(5..).string + assert_equal "5:*,1:4", SequenceSet.new(5..).append(1..4).string end test "#merge" do From 20df591133ce6ffd8adac31b2b50c0a21c460f1a Mon Sep 17 00:00:00 2001 From: nick evans Date: Fri, 17 Jan 2025 08:55:15 -0500 Subject: [PATCH 04/34] =?UTF-8?q?=F0=9F=90=9B=20Fix=20`SequenceSet#merge`?= =?UTF-8?q?=20with=20another=20SequenceSet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most methods convert their inputs into an array of range tuples. For efficiency, SequenceSet inputs just use the internal `@tuples` array directly. Unfortunately, the internal tuples arrays were also reused, which could cause a variety of bugs. Fortunately, the only bug I experienced was that adding a frozen SequenceSet would result in frozen tuples being added to a mutable set. But this could also result in modifications to one SequenceSet affecting another SequenceSet! --- lib/net/imap/sequence_set.rb | 4 ++-- test/net/imap/test_sequence_set.rb | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index b046b0eb..a3376cb4 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -1341,8 +1341,8 @@ def tuple_add(tuple) modifying! min, max = tuple lower, lower_idx = tuple_gte_with_index(min - 1) - if lower.nil? then tuples << tuple - elsif (max + 1) < lower.first then tuples.insert(lower_idx, tuple) + if lower.nil? then tuples << [min, max] + elsif (max + 1) < lower.first then tuples.insert(lower_idx, [min, max]) else tuple_coalesce(lower, lower_idx, min, max) end end diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index 94c096ba..17d804b2 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -362,6 +362,11 @@ def obj.to_sequence_set; 192_168.001_255 end assert_equal seqset["1:3,5,7:9"], seqset["1,3,5,7:8"].merge(seqset["2,8:9"]) assert_equal seqset["1:*"], seqset["5:*"].merge(1..4) assert_equal seqset["1:5"], seqset["1,3,5"].merge(seqset["2,4"]) + # when merging frozen SequenceSet + set = SequenceSet.new + set.merge SequenceSet[1, 3, 5] + set.merge SequenceSet[2..33] + assert_equal seqset[1..33], set end test "set - other" do From 7e53f60ea3b760667727a742a8ceb5040c5fa3f3 Mon Sep 17 00:00:00 2001 From: nick evans Date: Fri, 17 Jan 2025 09:37:37 -0500 Subject: [PATCH 05/34] =?UTF-8?q?=F0=9F=93=9A=20Fix=20`SequenceSet#cover?= =?UTF-8?q?=3F`=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `===` is not an alias for `cover?`. It delegates to `cover?` but handles errors differently. --- lib/net/imap/sequence_set.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index a3376cb4..654d5a1d 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -162,7 +162,7 @@ class IMAP # - #===: # Returns whether a given object is fully contained within +self+, or # +nil+ if the object cannot be converted to a compatible type. - # - #cover? (aliased as #===): + # - #cover?: # Returns whether a given object is fully contained within +self+. # - #intersect? (aliased as #overlap?): # Returns whether +self+ and a given object have any common elements. From 9215a1d12631147141d4f107bff5bfc1e045f3a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:57:43 +0000 Subject: [PATCH 06/34] :arrow_up: Bump step-security/harden-runner from 2.10.3 to 2.10.4 Bumps [step-security/harden-runner](https://github.com/step-security/harden-runner) from 2.10.3 to 2.10.4. - [Release notes](https://github.com/step-security/harden-runner/releases) - [Commits](https://github.com/step-security/harden-runner/compare/c95a14d0e5bab51a9f56296a4eb0e416910cd350...cb605e52c26070c328afc4562f0b4ada7618a84e) --- updated-dependencies: - dependency-name: step-security/harden-runner dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/push_gem.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push_gem.yml b/.github/workflows/push_gem.yml index e4a5f7ba..a085de5c 100644 --- a/.github/workflows/push_gem.yml +++ b/.github/workflows/push_gem.yml @@ -24,7 +24,7 @@ jobs: steps: # Set up - name: Harden Runner - uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 with: egress-policy: audit From 340d3e7043b71b75ac0a793364e475854f80b13e Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 22 Jan 2025 13:11:06 -0500 Subject: [PATCH 07/34] =?UTF-8?q?=F0=9F=94=A7=20ResponseParser=20config=20?= =?UTF-8?q?is=20mutable=20and=20non-global?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When ResponseParser was initialized without any explicit config, it used Config.global directly. This meant that changes to `parser.config` were also changes to the _global_ config! When ResponseParser is initialized with a config hash, that creates a frozen config object. This meant that changes to `parser.config` were impossible. So, when the given config is global or frozen, we create a new config and inherit from the given config. We want to continue using the given config as-is in other cases, so the client and its parser can share the same exact config. --- lib/net/imap/response_parser.rb | 6 +++- test/net/imap/test_imap_response_parser.rb | 35 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index f6dc438a..29ee38b8 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -13,13 +13,17 @@ class ResponseParser attr_reader :config - # :call-seq: Net::IMAP::ResponseParser.new -> Net::IMAP::ResponseParser + # Creates a new ResponseParser. + # + # When +config+ is frozen or global, the parser #config inherits from it. + # Otherwise, +config+ will be used directly. def initialize(config: Config.global) @str = nil @pos = nil @lex_state = nil @token = nil @config = Config[config] + @config = @config.new if @config == Config.global || @config.frozen? end # :call-seq: diff --git a/test/net/imap/test_imap_response_parser.rb b/test/net/imap/test_imap_response_parser.rb index e1eb16c5..5d594bca 100644 --- a/test/net/imap/test_imap_response_parser.rb +++ b/test/net/imap/test_imap_response_parser.rb @@ -118,6 +118,41 @@ def teardown # response data, should still use normal tests, below ############################################################################ + test "default config inherits from Config.global" do + parser = Net::IMAP::ResponseParser.new + refute parser.config.frozen? + refute_equal Net::IMAP::Config.global, parser.config + assert_same Net::IMAP::Config.global, parser.config.parent + end + + test "config can be passed in to #initialize" do + config = Net::IMAP::Config.global.new + parser = Net::IMAP::ResponseParser.new config: config + assert_same config, parser.config + end + + test "passing in global config inherits from Config.global" do + parser = Net::IMAP::ResponseParser.new config: Net::IMAP::Config.global + refute parser.config.frozen? + refute_equal Net::IMAP::Config.global, parser.config + assert_same Net::IMAP::Config.global, parser.config.parent + end + + test "config will inherits from passed in frozen config" do + parser = Net::IMAP::ResponseParser.new config: {debug: true} + refute_equal Net::IMAP::Config.global, parser.config.parent + refute parser.config.frozen? + + assert parser.config.parent.frozen? + assert parser.config.debug? + assert parser.config.inherited?(:debug) + + config = Net::IMAP::Config[debug: true] + parser = Net::IMAP::ResponseParser.new(config:) + refute_equal Net::IMAP::Config.global, parser.config.parent + assert_same config, parser.config.parent + end + # Strangly, there are no example responses for BINARY[section] in either # RFC3516 or RFC9051! The closest I found was RFC5259, and those examples # aren't FETCH responses. From 4cabe29c51454a2ca091a57bf3f74184ed8a0698 Mon Sep 17 00:00:00 2001 From: nick evans Date: Fri, 24 Jan 2025 09:52:30 -0500 Subject: [PATCH 08/34] =?UTF-8?q?=E2=9C=85=20Improve=20test=20coverage=20f?= =?UTF-8?q?or=20SequenceSet=20enums?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `#each_range` and `#each_number` were previously tested indirectly via `#ranges` and `#numbers`, but this adds tests on them directly. --- test/net/imap/test_sequence_set.rb | 41 +++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index 17d804b2..33e7388b 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -797,10 +797,42 @@ def test_inspect((expected, input, freeze)) assert_equal data[:entries], seqset.each_entry.to_a end + test "#each_range" do |data| + seqset = SequenceSet.new(data[:input]) + array = [] + assert_equal seqset, seqset.each_range { array << _1 } + assert_equal data[:ranges], array + assert_equal data[:ranges], seqset.each_range.to_a + end + test "#ranges" do |data| assert_equal data[:ranges], SequenceSet.new(data[:input]).ranges end + test "#each_number" do |data| + seqset = SequenceSet.new(data[:input]) + expected = data[:numbers] + enum = seqset.each_number + if expected.is_a?(Class) && expected < Exception + assert_raise expected do enum.to_a end + assert_raise expected do enum.each do fail "shouldn't get here" end end + else + array = [] + assert_equal seqset, seqset.each_number { array << _1 } + assert_equal expected, array + assert_equal expected, seqset.each_number.to_a + end + end + + test "#numbers" do |data| + expected = data[:numbers] + if expected.is_a?(Class) && expected < Exception + assert_raise expected do SequenceSet.new(data[:input]).numbers end + else + assert_equal expected, SequenceSet.new(data[:input]).numbers + end + end + test "#string" do |data| set = SequenceSet.new(data[:input]) str = data[:to_s] @@ -862,15 +894,6 @@ def test_inspect((expected, input, freeze)) assert_equal(data[:complement], (~set).to_s) end - test "#numbers" do |data| - expected = data[:numbers] - if expected.is_a?(Class) && expected < Exception - assert_raise expected do SequenceSet.new(data[:input]).numbers end - else - assert_equal expected, SequenceSet.new(data[:input]).numbers - end - end - test "SequenceSet[input]" do |input| case (input = data[:input]) when nil From db83bec1e985bfcbbb82b74970fe572e733de79a Mon Sep 17 00:00:00 2001 From: nick evans Date: Fri, 24 Jan 2025 17:34:02 -0500 Subject: [PATCH 09/34] =?UTF-8?q?=F0=9F=93=9A=20Update=20SequenceSet=20"Wh?= =?UTF-8?q?at's=20Here=3F"=20index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This splits off "normalized" vs "order preserving" methods in the "What's here?" sections. --- lib/net/imap/sequence_set.rb | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index 654d5a1d..ba3af4e3 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -200,14 +200,11 @@ class IMAP # # === Methods for Iterating # + # Normalized (sorted and coalesced): # - #each_element: Yields each number and range in the set, sorted and # coalesced, and returns +self+. # - #elements (aliased as #to_a): Returns an Array of every number and range # in the set, sorted and coalesced. - # - #each_entry: Yields each number and range in the set, unsorted and - # without deduplicating numbers or coalescing ranges, and returns +self+. - # - #entries: Returns an Array of every number and range in the set, - # unsorted and without deduplicating numbers or coalescing ranges. # - #each_range: # Yields each element in the set as a Range and returns +self+. # - #ranges: Returns an Array of every element in the set, converting @@ -217,6 +214,12 @@ class IMAP # ranges into all of their contained numbers. # - #to_set: Returns a Set containing all of the #numbers in the set. # + # Order preserving: + # - #each_entry: Yields each number and range in the set, unsorted and + # without deduplicating numbers or coalescing ranges, and returns +self+. + # - #entries: Returns an Array of every number and range in the set, + # unsorted and without deduplicating numbers or coalescing ranges. + # # === Methods for \Set Operations # These methods do not modify +self+. # @@ -236,19 +239,29 @@ class IMAP # === Methods for Assigning # These methods add or replace elements in +self+. # + # Normalized (sorted and coalesced): + # + # These methods always update #string to be fully sorted and coalesced. + # # - #add (aliased as #<<): Adds a given object to the set; returns +self+. # - #add?: If the given object is not an element in the set, adds it and # returns +self+; otherwise, returns +nil+. # - #merge: Merges multiple elements into the set; returns +self+. + # - #complement!: Replaces the contents of the set with its own #complement. + # + # Order preserving: + # + # These methods _may_ cause #string to not be sorted or coalesced. + # # - #append: Adds a given object to the set, appending it to the existing # string, and returns +self+. # - #string=: Assigns a new #string value and replaces #elements to match. # - #replace: Replaces the contents of the set with the contents # of a given object. - # - #complement!: Replaces the contents of the set with its own #complement. # # === Methods for Deleting - # These methods remove elements from +self+. + # These methods remove elements from +self+, and update #string to be fully + # sorted and coalesced. # # - #clear: Removes all elements in the set; returns +self+. # - #delete: Removes a given object from the set; returns +self+. From dab349277a9beddf48b9852fb673211beefa7e8f Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 6 Jan 2025 21:31:24 -0500 Subject: [PATCH 10/34] =?UTF-8?q?=E2=9C=A8=20Add=20`SequenceSet#count=5Fwi?= =?UTF-8?q?th=5Fduplicates`,=20etc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds three new SequenceSet methods for querying about "duplicates": * `#has_duplicates?`: Returns whether the ordered entries contain any duplicates. * `#count_duplicates`: Returns the count of duplicates in the ordered entries. * `#count_with_duplicates`: Returns the count of numbers in the ordered entries, including any repeated numbers. This is useful for adding guards to `CopyUIDData` and `UIDPlusData`, and for getting the size of `SequenceSet#each_ordered_number` (another PR). --- lib/net/imap/sequence_set.rb | 63 +++++++++++++++++++++++++++--- test/net/imap/test_sequence_set.rb | 21 ++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index ba3af4e3..b83c9f84 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -198,6 +198,14 @@ class IMAP # - #full?: Returns whether the set contains every possible value, including # *. # + # Denormalized properties: + # - #has_duplicates?: Returns whether the ordered entries repeat any + # numbers. + # - #count_duplicates: Returns the count of repeated numbers in the ordered + # entries. + # - #count_with_duplicates: Returns the count of numbers in the ordered + # entries, including any repeated numbers. + # # === Methods for Iterating # # Normalized (sorted and coalesced): @@ -923,9 +931,7 @@ def numbers; each_number.to_a end # Related: #entries, #each_element def each_entry(&block) # :yields: integer or range or :* return to_enum(__method__) unless block_given? - return each_element(&block) unless @string - @string.split(",").each do yield tuple_to_entry str_to_tuple _1 end - self + each_entry_tuple do yield tuple_to_entry _1 end end # Yields each number or range (or :*) in #elements to the block @@ -943,6 +949,16 @@ def each_element # :yields: integer or range or :* private + def each_entry_tuple(&block) + return to_enum(__method__) unless block_given? + if @string + @string.split(",") do block.call str_to_tuple _1 end + else + @tuples.each(&block) + end + self + end + def tuple_to_entry((min, max)) if min == STAR_INT then :* elsif max == STAR_INT then min.. @@ -1001,12 +1017,49 @@ def to_set; Set.new(numbers) end # If * and 2**32 - 1 (the maximum 32-bit unsigned # integer value) are both in the set, they will only be counted once. def count - @tuples.sum(@tuples.count) { _2 - _1 } + - (include_star? && include?(UINT32_MAX) ? -1 : 0) + count_numbers_in_tuples(@tuples) end alias size count + # Returns the count of numbers in the ordered #entries, including any + # repeated numbers. + # + # When #string is normalized, this behaves the same as #count. + # + # Related: #entries, #count_duplicates, #has_duplicates? + def count_with_duplicates + return count unless @string + count_numbers_in_tuples(each_entry_tuple) + end + + # Returns the count of repeated numbers in the ordered #entries. + # + # When #string is normalized, this is zero. + # + # Related: #entries, #count_with_duplicates, #has_duplicates? + def count_duplicates + return 0 unless @string + count_with_duplicates - count + end + + # :call-seq: has_duplicates? -> true | false + # + # Returns whether or not the ordered #entries repeat any numbers. + # + # Always returns +false+ when #string is normalized. + # + # Related: #entries, #count_with_duplicates, #count_duplicates? + def has_duplicates? + return false unless @string + count_with_duplicates != count + end + + private def count_numbers_in_tuples(tuples) + tuples.sum(tuples.count) { _2 - _1 } + + (include_star? && include?(UINT32_MAX) ? -1 : 0) + end + # Returns the index of +number+ in the set, or +nil+ if +number+ isn't in # the set. # diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index 33e7388b..971a81fd 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -710,6 +710,7 @@ def test_inspect((expected, input, freeze)) to_s: "1:5,3:7,10:9,10:11", normalize: "1:7,9:11", count: 10, + count_dups: 4, complement: "8,12:*", }, keep: true @@ -722,6 +723,7 @@ def test_inspect((expected, input, freeze)) to_s: "1:5,3:4,9:11,10", normalize: "1:5,9:11", count: 8, + count_dups: 3, complement: "6:8,12:*", }, keep: true @@ -878,6 +880,25 @@ def test_inspect((expected, input, freeze)) assert_equal data[:count], SequenceSet.new(data[:input]).count end + test "#count_with_duplicates" do |data| + dups = data[:count_dups] || 0 + count = data[:count] + dups + seqset = SequenceSet.new(data[:input]) + assert_equal count, seqset.count_with_duplicates + end + + test "#count_duplicates" do |data| + dups = data[:count_dups] || 0 + seqset = SequenceSet.new(data[:input]) + assert_equal dups, seqset.count_duplicates + end + + test "#has_duplicates?" do |data| + has_dups = !(data[:count_dups] || 0).zero? + seqset = SequenceSet.new(data[:input]) + assert_equal has_dups, seqset.has_duplicates? + end + test "#valid_string" do |data| if (expected = data[:to_s]).empty? assert_raise DataFormatError do From c26dc510c2e76227cbafde3def293fe555ca4ea8 Mon Sep 17 00:00:00 2001 From: nick evans Date: Fri, 24 Jan 2025 18:05:26 -0500 Subject: [PATCH 11/34] =?UTF-8?q?=E2=99=BB=EF=B8=8F=E2=9C=85=20Refactor=20?= =?UTF-8?q?SequenceSet=20enumerator=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Less duplication, still fairly easy to read, IMO. --- test/net/imap/test_sequence_set.rb | 46 +++++++++++++++++------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index 971a81fd..436eef53 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -779,12 +779,22 @@ def test_inspect((expected, input, freeze)) assert_equal data[:elements], SequenceSet.new(data[:input]).elements end - test "#each_element" do |data| - seqset = SequenceSet.new(data[:input]) + def assert_seqset_enum(expected, seqset, enum) + array = [] + assert_equal seqset, seqset.send(enum) { array << _1 } + assert_equal expected, array + array = [] - assert_equal seqset, seqset.each_element { array << _1 } - assert_equal data[:elements], array - assert_equal data[:elements], seqset.each_element.to_a + assert_equal seqset, seqset.send(enum).each { array << _1 } + assert_equal expected, array + + assert_equal expected, seqset.send(enum).to_a + end + + test "#each_element" do |data| + seqset = SequenceSet.new(data[:input]) + expected = data[:elements] + assert_seqset_enum expected, seqset, :each_element end test "#entries" do |data| @@ -792,19 +802,15 @@ def test_inspect((expected, input, freeze)) end test "#each_entry" do |data| - seqset = SequenceSet.new(data[:input]) - array = [] - assert_equal seqset, seqset.each_entry { array << _1 } - assert_equal data[:entries], array - assert_equal data[:entries], seqset.each_entry.to_a + seqset = SequenceSet.new(data[:input]) + expected = data[:entries] + assert_seqset_enum expected, seqset, :each_entry end test "#each_range" do |data| - seqset = SequenceSet.new(data[:input]) - array = [] - assert_equal seqset, seqset.each_range { array << _1 } - assert_equal data[:ranges], array - assert_equal data[:ranges], seqset.each_range.to_a + seqset = SequenceSet.new(data[:input]) + expected = data[:ranges] + assert_seqset_enum expected, seqset, :each_range end test "#ranges" do |data| @@ -814,15 +820,15 @@ def test_inspect((expected, input, freeze)) test "#each_number" do |data| seqset = SequenceSet.new(data[:input]) expected = data[:numbers] - enum = seqset.each_number if expected.is_a?(Class) && expected < Exception + assert_raise expected do + seqset.each_number do fail "shouldn't get here" end + end + enum = seqset.each_number assert_raise expected do enum.to_a end assert_raise expected do enum.each do fail "shouldn't get here" end end else - array = [] - assert_equal seqset, seqset.each_number { array << _1 } - assert_equal expected, array - assert_equal expected, seqset.each_number.to_a + assert_seqset_enum expected, seqset, :each_number end end From a2cc018a7fce6d1eccb67fd45c232abf52fcf657 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sun, 19 Jan 2025 23:26:09 -0500 Subject: [PATCH 12/34] =?UTF-8?q?=E2=9C=A8=20Add=20`SequenceSet#each=5Ford?= =?UTF-8?q?ered=5Fnumber`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Yields each number in the ordered entries and returns +self+. --- lib/net/imap/sequence_set.rb | 33 +++++++++++++++++++++++------- test/net/imap/test_sequence_set.rb | 18 ++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index b83c9f84..34c7c198 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -227,6 +227,8 @@ class IMAP # without deduplicating numbers or coalescing ranges, and returns +self+. # - #entries: Returns an Array of every number and range in the set, # unsorted and without deduplicating numbers or coalescing ranges. + # - #each_ordered_number: Yields each number in the ordered entries and + # returns +self+. # # === Methods for \Set Operations # These methods do not modify +self+. @@ -990,19 +992,36 @@ def each_range # :yields: range # Returns an enumerator when called without a block (even if the set # contains *). # - # Related: #numbers + # Related: #numbers, #each_ordered_number def each_number(&block) # :yields: integer return to_enum(__method__) unless block_given? raise RangeError, '%s contains "*"' % [self.class] if include_star? - each_element do |elem| - case elem - when Range then elem.each(&block) - when Integer then block.(elem) - end - end + @tuples.each do each_number_in_tuple _1, _2, &block end self end + # Yields each number in #entries to the block and returns self. + # If the set contains a *, RangeError will be raised. + # + # Returns an enumerator when called without a block (even if the set + # contains *). + # + # Related: #entries, #each_number + def each_ordered_number(&block) + return to_enum(__method__) unless block_given? + raise RangeError, '%s contains "*"' % [self.class] if include_star? + each_entry_tuple do each_number_in_tuple _1, _2, &block end + end + + private def each_number_in_tuple(min, max, &block) + if min == STAR_INT then yield :* + elsif min == max then yield min + elsif max != STAR_INT then (min..max).each(&block) + else + raise RangeError, "#{SequenceSet} cannot enumerate range with '*'" + end + end + # Returns a Set with all of the #numbers in the sequence set. # # If the set contains a *, RangeError will be raised. diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index 436eef53..71d62997 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -683,6 +683,7 @@ def test_inspect((expected, input, freeze)) entries: [46, 6..7, 15, 1..3], ranges: [1..3, 6..7, 15..15, 46..46], numbers: [1, 2, 3, 6, 7, 15, 46], + ordered: [46, 6, 7, 15, 1, 2, 3], to_s: "46,7:6,15,3:1", normalize: "1:3,6:7,15,46", count: 7, @@ -707,6 +708,7 @@ def test_inspect((expected, input, freeze)) entries: [1..5, 3..7, 9..10, 10..11], ranges: [1..7, 9..11], numbers: [1, 2, 3, 4, 5, 6, 7, 9, 10, 11], + ordered: [1,2,3,4,5, 3,4,5,6,7, 9,10, 10,11], to_s: "1:5,3:7,10:9,10:11", normalize: "1:7,9:11", count: 10, @@ -720,6 +722,7 @@ def test_inspect((expected, input, freeze)) entries: [1..5, 3..4, 9..11, 10], ranges: [1..5, 9..11], numbers: [1, 2, 3, 4, 5, 9, 10, 11], + ordered: [1,2,3,4,5, 3,4, 9,10,11, 10], to_s: "1:5,3:4,9:11,10", normalize: "1:5,9:11", count: 8, @@ -832,6 +835,21 @@ def assert_seqset_enum(expected, seqset, enum) end end + test "#each_ordered_number" do |data| + seqset = SequenceSet.new(data[:input]) + expected = data[:ordered] || data[:numbers] + if expected.is_a?(Class) && expected < Exception + assert_raise expected do + seqset.each_ordered_number do fail "shouldn't get here" end + end + enum = seqset.each_ordered_number + assert_raise expected do enum.to_a end + assert_raise expected do enum.each do fail "shouldn't get here" end end + else + assert_seqset_enum expected, seqset, :each_ordered_number + end + end + test "#numbers" do |data| expected = data[:numbers] if expected.is_a?(Class) && expected < Exception From 63fbdd975d71e152adc0c61a27b9c70b38a0a011 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sun, 26 Jan 2025 21:49:47 -0500 Subject: [PATCH 13/34] =?UTF-8?q?=F0=9F=90=9B=20Fix=20SequenceSet=20count?= =?UTF-8?q?=20dups=20with=20multiple=20"*"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In `#count`, "*" is treated as if it is effectively UINT32_MAX. That was also the intention for `#count_with_duplicates`. Unlike `#count`, which can assume that `*` only appears at most once, `#count_with_duplicates` needs to check each entry. This means that, e.g: SequenceSet["#{UINT32_MAX}:*"].count_with_duplicates == 1 SequenceSet["#{UINT32_MAX},*"].count_with_duplicates == 2 --- lib/net/imap/sequence_set.rb | 24 ++++++++++++++---------- test/net/imap/test_sequence_set.rb | 13 +++++++++++++ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index 34c7c198..1d09aa05 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -1033,10 +1033,13 @@ def to_set; Set.new(numbers) end # Returns the count of #numbers in the set. # - # If * and 2**32 - 1 (the maximum 32-bit unsigned - # integer value) are both in the set, they will only be counted once. + # * will be counted as 2**32 - 1 (the maximum 32-bit + # unsigned integer value). + # + # Related: #count_with_duplicates def count - count_numbers_in_tuples(@tuples) + @tuples.sum(@tuples.count) { _2 - _1 } + + (include_star? && include?(UINT32_MAX) ? -1 : 0) end alias size count @@ -1044,15 +1047,21 @@ def count # Returns the count of numbers in the ordered #entries, including any # repeated numbers. # + # * will be counted as 2**32 - 1 (the maximum 32-bit + # unsigned integer value). + # # When #string is normalized, this behaves the same as #count. # # Related: #entries, #count_duplicates, #has_duplicates? def count_with_duplicates return count unless @string - count_numbers_in_tuples(each_entry_tuple) + each_entry_tuple.sum {|min, max| + max - min + ((max == STAR_INT && min != STAR_INT) ? 0 : 1) + } end - # Returns the count of repeated numbers in the ordered #entries. + # Returns the count of repeated numbers in the ordered #entries, the + # difference between #count_with_duplicates and #count. # # When #string is normalized, this is zero. # @@ -1074,11 +1083,6 @@ def has_duplicates? count_with_duplicates != count end - private def count_numbers_in_tuples(tuples) - tuples.sum(tuples.count) { _2 - _1 } + - (include_star? && include?(UINT32_MAX) ? -1 : 0) - end - # Returns the index of +number+ in the set, or +nil+ if +number+ isn't in # the set. # diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index 71d62997..b6995305 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -730,6 +730,19 @@ def test_inspect((expected, input, freeze)) complement: "6:8,12:*", }, keep: true + data "multiple *", { + input: "2:*,3:*,*", + elements: [2..], + entries: [2.., 3.., :*], + ranges: [2..], + numbers: RangeError, + to_s: "2:*,3:*,*", + normalize: "2:*", + count: 2**32 - 2, + count_dups: 2**32 - 2, + complement: "1", + }, keep: true + data "array", { input: ["1:5,3:4", 9..11, "10", 99, :*], elements: [1..5, 9..11, 99, :*], From a55269aa8594bfad2e6bcef12523186c6a92ac5e Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 27 Jan 2025 11:10:02 -0500 Subject: [PATCH 14/34] =?UTF-8?q?=E2=9E=95=20Add=20"irb"=20to=20Gemfile=20?= =?UTF-8?q?to=20silence=20warning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gemfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile b/Gemfile index cbceddc9..4b1dfca8 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ gem "digest" gem "strscan" gem "base64" +gem "irb" gem "rake" gem "rdoc" gem "test-unit" From 8850cf4be07180c70f404f745c9623a7ea1d4d68 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Thu, 30 Jan 2025 14:27:17 +0900 Subject: [PATCH 15/34] Extend timeout because it's flaky with macOS --- test/net/imap/fake_server/test_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net/imap/fake_server/test_helper.rb b/test/net/imap/fake_server/test_helper.rb index eee6a153..1987d048 100644 --- a/test/net/imap/fake_server/test_helper.rb +++ b/test/net/imap/fake_server/test_helper.rb @@ -4,7 +4,7 @@ module Net::IMAP::FakeServer::TestHelper - def run_fake_server_in_thread(ignore_io_error: false, timeout: 5, **opts) + def run_fake_server_in_thread(ignore_io_error: false, timeout: 10, **opts) Timeout.timeout(timeout) do server = Net::IMAP::FakeServer.new(timeout: timeout, **opts) @threads << Thread.new do From 29aa826936c977c6ddc8183cca355b0eb65b5529 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Thu, 30 Jan 2025 14:19:36 +0900 Subject: [PATCH 16/34] Omit test_imaps_verify_none and test_imaps_with_ca_file if it catched OpenSSL::SSL::SSLError and running with macOS --- test/net/imap/test_imap.rb | 40 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/test/net/imap/test_imap.rb b/test/net/imap/test_imap.rb index f3e53c40..10a29c40 100644 --- a/test/net/imap/test_imap.rb +++ b/test/net/imap/test_imap.rb @@ -46,14 +46,18 @@ def test_imaps_with_ca_file # Otherwise, failures can't logout and need to wait for the timeout. verified, imap = :unknown, nil assert_nothing_raised do - imaps_test do |port| - imap = Net::IMAP.new("localhost", - port: port, - ssl: { :ca_file => CA_FILE }) - verified = imap.tls_verified? - imap - rescue SystemCallError - skip $! + begin + imaps_test do |port| + imap = Net::IMAP.new("localhost", + port: port, + ssl: { :ca_file => CA_FILE }) + verified = imap.tls_verified? + imap + rescue SystemCallError + skip $! + end + rescue OpenSSL::SSL::SSLError => e + raise e unless /darwin/ =~ RUBY_PLATFORM end end assert_equal true, verified @@ -69,14 +73,18 @@ def test_imaps_verify_none # Otherwise, failures can't logout and need to wait for the timeout. verified, imap = :unknown, nil assert_nothing_raised do - imaps_test do |port| - imap = Net::IMAP.new( - server_addr, - port: port, - ssl: { :verify_mode => OpenSSL::SSL::VERIFY_NONE } - ) - verified = imap.tls_verified? - imap + begin + imaps_test do |port| + imap = Net::IMAP.new( + server_addr, + port: port, + ssl: { :verify_mode => OpenSSL::SSL::VERIFY_NONE } + ) + verified = imap.tls_verified? + imap + end + rescue OpenSSL::SSL::SSLError => e + raise e unless /darwin/ =~ RUBY_PLATFORM end end assert_equal false, verified From 6d27483f7dd4f90749daa2f06a9db06780250333 Mon Sep 17 00:00:00 2001 From: nick evans Date: Thu, 23 Jan 2025 11:04:12 -0500 Subject: [PATCH 17/34] =?UTF-8?q?=F0=9F=9A=9A=20Move=20"response=20data"?= =?UTF-8?q?=20tests=20to=20appropriate=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/net/imap/test_imap_response_data.rb | 58 ---------------------- test/net/imap/test_imap_response_parser.rb | 21 ++++++++ test/net/imap/test_thread_member.rb | 27 ++++++++++ 3 files changed, 48 insertions(+), 58 deletions(-) delete mode 100644 test/net/imap/test_imap_response_data.rb create mode 100644 test/net/imap/test_thread_member.rb diff --git a/test/net/imap/test_imap_response_data.rb b/test/net/imap/test_imap_response_data.rb deleted file mode 100644 index 0bee8b9a..00000000 --- a/test/net/imap/test_imap_response_data.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -require "net/imap" -require "test/unit" - -class IMAPResponseDataTest < Test::Unit::TestCase - - def setup - Net::IMAP.config.reset - @do_not_reverse_lookup = Socket.do_not_reverse_lookup - Socket.do_not_reverse_lookup = true - end - - def teardown - Socket.do_not_reverse_lookup = @do_not_reverse_lookup - end - - def test_uidplus_copyuid__uid_mapping - parser = Net::IMAP::ResponseParser.new - response = parser.parse( - "A004 OK [copyUID 9999 20:19,500:495 92:97,101:100] Done\r\n" - ) - code = response.data.code - assert_equal( - { - 19 => 92, - 20 => 93, - 495 => 94, - 496 => 95, - 497 => 96, - 498 => 97, - 499 => 100, - 500 => 101, - }, - code.data.uid_mapping - ) - end - - def test_thread_member_to_sequence_set - # copied from the fourth example in RFC5256: (3 6 (4 23)(44 7 96)) - thmember = Net::IMAP::ThreadMember.method :new - thread = thmember.(3, [ - thmember.(6, [ - thmember.(4, [ - thmember.(23, []) - ]), - thmember.(44, [ - thmember.(7, [ - thmember.(96, []) - ]) - ]) - ]) - ]) - expected = Net::IMAP::SequenceSet.new("3:4,6:7,23,44,96") - assert_equal(expected, thread.to_sequence_set) - end - -end diff --git a/test/net/imap/test_imap_response_parser.rb b/test/net/imap/test_imap_response_parser.rb index 5d594bca..9bcced4d 100644 --- a/test/net/imap/test_imap_response_parser.rb +++ b/test/net/imap/test_imap_response_parser.rb @@ -202,4 +202,25 @@ def test_fetch_binary_and_binary_size Net::IMAP.debug = debug end + test "COPYUID with backwards ranges" do + parser = Net::IMAP::ResponseParser.new + response = parser.parse( + "A004 OK [copyUID 9999 20:19,500:495 92:97,101:100] Done\r\n" + ) + code = response.data.code + assert_equal( + { + 19 => 92, + 20 => 93, + 495 => 94, + 496 => 95, + 497 => 96, + 498 => 97, + 499 => 100, + 500 => 101, + }, + code.data.uid_mapping + ) + end + end diff --git a/test/net/imap/test_thread_member.rb b/test/net/imap/test_thread_member.rb new file mode 100644 index 00000000..afa933fb --- /dev/null +++ b/test/net/imap/test_thread_member.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "net/imap" +require "test/unit" + +class ThreadMemberTest < Test::Unit::TestCase + + test "#to_sequence_set" do + # copied from the fourth example in RFC5256: (3 6 (4 23)(44 7 96)) + thmember = Net::IMAP::ThreadMember.method :new + thread = thmember.(3, [ + thmember.(6, [ + thmember.(4, [ + thmember.(23, []) + ]), + thmember.(44, [ + thmember.(7, [ + thmember.(96, []) + ]) + ]) + ]) + ]) + expected = Net::IMAP::SequenceSet.new("3:4,6:7,23,44,96") + assert_equal(expected, thread.to_sequence_set) + end + +end From 3d52d84f3268b03396bc37b60596fd5439afa281 Mon Sep 17 00:00:00 2001 From: nick evans Date: Thu, 23 Jan 2025 10:56:17 -0500 Subject: [PATCH 18/34] =?UTF-8?q?=E2=9C=85=20Improve=20UIDPlusData#uid=5Fm?= =?UTF-8?q?apping=20test=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explicitly test how unsorted uid-sets are handled. --- test/net/imap/test_uid_plus_data.rb | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/net/imap/test_uid_plus_data.rb diff --git a/test/net/imap/test_uid_plus_data.rb b/test/net/imap/test_uid_plus_data.rb new file mode 100644 index 00000000..210a000e --- /dev/null +++ b/test/net/imap/test_uid_plus_data.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "net/imap" +require "test/unit" + +class TestUIDPlusData < Test::Unit::TestCase + + test "#uid_mapping with sorted source_uids" do + uidplus = Net::IMAP::UIDPlusData.new( + 1, [19, 20, *(495..500)], [*(92..97), 100, 101], + ) + assert_equal( + { + 19 => 92, + 20 => 93, + 495 => 94, + 496 => 95, + 497 => 96, + 498 => 97, + 499 => 100, + 500 => 101, + }, + uidplus.uid_mapping + ) + end + + test "#uid_mapping for with source_uids in unsorted order" do + uidplus = Net::IMAP::UIDPlusData.new( + 1, [*(495..500), 19, 20], [*(92..97), 100, 101], + ) + assert_equal( + { + 495 => 92, + 496 => 93, + 497 => 94, + 498 => 95, + 499 => 96, + 500 => 97, + 19 => 100, + 20 => 101, + }, + uidplus.uid_mapping + ) + end + +end From 7bfffe8e3bff8923bc495377ad4a5d6ed9dfe82d Mon Sep 17 00:00:00 2001 From: nick evans Date: Sat, 18 Jan 2025 11:49:11 -0500 Subject: [PATCH 19/34] =?UTF-8?q?=F0=9F=9A=9A=20Move=20UIDPlusData=20to=20?= =?UTF-8?q?its=20own=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/response_data.rb | 50 +------------------------------ lib/net/imap/uidplus_data.rb | 56 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 49 deletions(-) create mode 100644 lib/net/imap/uidplus_data.rb diff --git a/lib/net/imap/response_data.rb b/lib/net/imap/response_data.rb index 16ff306c..40586ebd 100644 --- a/lib/net/imap/response_data.rb +++ b/lib/net/imap/response_data.rb @@ -7,6 +7,7 @@ class IMAP < Protocol autoload :UIDFetchData, "#{__dir__}/fetch_data" autoload :SearchResult, "#{__dir__}/search_result" autoload :SequenceSet, "#{__dir__}/sequence_set" + autoload :UIDPlusData, "#{__dir__}/uidplus_data" autoload :VanishedData, "#{__dir__}/vanished_data" # Net::IMAP::ContinuationRequest represents command continuation requests. @@ -344,55 +345,6 @@ class ResponseCode < Struct.new(:name, :data) # code data can take. end - # UIDPlusData represents the ResponseCode#data that accompanies the - # +APPENDUID+ and +COPYUID+ {response codes}[rdoc-ref:ResponseCode]. - # - # A server that supports +UIDPLUS+ should send a UIDPlusData object inside - # every TaggedResponse returned by the append[rdoc-ref:Net::IMAP#append], - # copy[rdoc-ref:Net::IMAP#copy], move[rdoc-ref:Net::IMAP#move], {uid - # copy}[rdoc-ref:Net::IMAP#uid_copy], and {uid - # move}[rdoc-ref:Net::IMAP#uid_move] commands---unless the destination - # mailbox reports +UIDNOTSTICKY+. - # - # == Required capability - # Requires either +UIDPLUS+ [RFC4315[https://www.rfc-editor.org/rfc/rfc4315]] - # or +IMAP4rev2+ capability. - # - class UIDPlusData < Struct.new(:uidvalidity, :source_uids, :assigned_uids) - ## - # method: uidvalidity - # :call-seq: uidvalidity -> nonzero uint32 - # - # The UIDVALIDITY of the destination mailbox. - - ## - # method: source_uids - # :call-seq: source_uids -> nil or an array of nonzero uint32 - # - # The UIDs of the copied or moved messages. - # - # Note:: Returns +nil+ for Net::IMAP#append. - - ## - # method: assigned_uids - # :call-seq: assigned_uids -> an array of nonzero uint32 - # - # The newly assigned UIDs of the copied, moved, or appended messages. - # - # Note:: This always returns an array, even when it contains only one UID. - - ## - # :call-seq: uid_mapping -> nil or a hash - # - # Returns a hash mapping each source UID to the newly assigned destination - # UID. - # - # Note:: Returns +nil+ for Net::IMAP#append. - def uid_mapping - source_uids&.zip(assigned_uids)&.to_h - end - end - # MailboxList represents the data of an untagged +LIST+ response, for a # _single_ mailbox path. IMAP#list returns an array of MailboxList objects. # diff --git a/lib/net/imap/uidplus_data.rb b/lib/net/imap/uidplus_data.rb new file mode 100644 index 00000000..2c478e2c --- /dev/null +++ b/lib/net/imap/uidplus_data.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Net + class IMAP < Protocol + + # UIDPlusData represents the ResponseCode#data that accompanies the + # +APPENDUID+ and +COPYUID+ {response codes}[rdoc-ref:ResponseCode]. + # + # A server that supports +UIDPLUS+ should send a UIDPlusData object inside + # every TaggedResponse returned by the append[rdoc-ref:Net::IMAP#append], + # copy[rdoc-ref:Net::IMAP#copy], move[rdoc-ref:Net::IMAP#move], {uid + # copy}[rdoc-ref:Net::IMAP#uid_copy], and {uid + # move}[rdoc-ref:Net::IMAP#uid_move] commands---unless the destination + # mailbox reports +UIDNOTSTICKY+. + # + # == Required capability + # Requires either +UIDPLUS+ [RFC4315[https://www.rfc-editor.org/rfc/rfc4315]] + # or +IMAP4rev2+ capability. + # + class UIDPlusData < Struct.new(:uidvalidity, :source_uids, :assigned_uids) + ## + # method: uidvalidity + # :call-seq: uidvalidity -> nonzero uint32 + # + # The UIDVALIDITY of the destination mailbox. + + ## + # method: source_uids + # :call-seq: source_uids -> nil or an array of nonzero uint32 + # + # The UIDs of the copied or moved messages. + # + # Note:: Returns +nil+ for Net::IMAP#append. + + ## + # method: assigned_uids + # :call-seq: assigned_uids -> an array of nonzero uint32 + # + # The newly assigned UIDs of the copied, moved, or appended messages. + # + # Note:: This always returns an array, even when it contains only one UID. + + ## + # :call-seq: uid_mapping -> nil or a hash + # + # Returns a hash mapping each source UID to the newly assigned destination + # UID. + # + # Note:: Returns +nil+ for Net::IMAP#append. + def uid_mapping + source_uids&.zip(assigned_uids)&.to_h + end + end + + end +end From d51d12ea0ae6ea593ea63577790fa72a515e3875 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sun, 19 Jan 2025 10:15:14 -0500 Subject: [PATCH 20/34] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Parse=20`uid-set`=20?= =?UTF-8?q?as=20`sequence-set`=20without=20`*`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In addition to letting us delete some code, this is also a step towards replacing `UIDPlusData` with new (incompatible) data structures that store UID sets directly, rather than converted into arrays of integers. --- lib/net/imap/response_parser.rb | 20 ++++++++++---------- test/net/imap/test_imap_response_parser.rb | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index 29ee38b8..03f135ec 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -2005,7 +2005,7 @@ def charset__list def resp_code_apnd__data validity = number; SP! dst_uids = uid_set # uniqueid ⊂ uid-set - UIDPlusData.new(validity, nil, dst_uids) + UIDPlus(validity, nil, dst_uids) end # already matched: "COPYUID" @@ -2015,6 +2015,12 @@ def resp_code_copy__data validity = number; SP! src_uids = uid_set; SP! dst_uids = uid_set + UIDPlus(validity, src_uids, dst_uids) + end + + def UIDPlus(validity, src_uids, dst_uids) + src_uids &&= src_uids.each_ordered_number.to_a + dst_uids = dst_uids.each_ordered_number.to_a UIDPlusData.new(validity, src_uids, dst_uids) end @@ -2141,15 +2147,9 @@ def nparens__objectid; NIL? ? nil : parens__objectid end # uniqueid = nz-number # ; Strictly ascending def uid_set - token = match(T_NUMBER, T_ATOM) - case token.symbol - when T_NUMBER then [Integer(token.value)] - when T_ATOM - token.value.split(",").flat_map {|range| - range = range.split(":").map {|uniqueid| Integer(uniqueid) } - range.size == 1 ? range : Range.new(range.min, range.max).to_a - } - end + set = sequence_set + parse_error("uid-set cannot contain '*'") if set.include_star? + set end def nil_atom diff --git a/test/net/imap/test_imap_response_parser.rb b/test/net/imap/test_imap_response_parser.rb index 9bcced4d..b24c63ff 100644 --- a/test/net/imap/test_imap_response_parser.rb +++ b/test/net/imap/test_imap_response_parser.rb @@ -202,6 +202,15 @@ def test_fetch_binary_and_binary_size Net::IMAP.debug = debug end + test "APPENDUID with '*'" do + parser = Net::IMAP::ResponseParser.new + assert_raise_with_message Net::IMAP::ResponseParseError, /uid-set cannot contain '\*'/ do + parser.parse( + "A004 OK [appendUID 1 1:*] Done\r\n" + ) + end + end + test "COPYUID with backwards ranges" do parser = Net::IMAP::ResponseParser.new response = parser.parse( @@ -223,4 +232,13 @@ def test_fetch_binary_and_binary_size ) end + test "COPYUID with '*'" do + parser = Net::IMAP::ResponseParser.new + assert_raise_with_message Net::IMAP::ResponseParseError, /uid-set cannot contain '\*'/ do + parser.parse( + "A004 OK [copyUID 1 1:* 1:*] Done\r\n" + ) + end + end + end From 97d3a21aca738762fceaf138333d24c40c9266f8 Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 3 Feb 2025 09:43:17 -0500 Subject: [PATCH 21/34] =?UTF-8?q?=F0=9F=93=9A=20Update=20SequenceSet=20rdo?= =?UTF-8?q?c=20for=20sorted=20set=20indexing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/sequence_set.rb | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index 1d09aa05..8f9f7d38 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -183,11 +183,11 @@ class IMAP # - #max: Returns the maximum number in the set. # - #minmax: Returns the minimum and maximum numbers in the set. # - # Accessing value by (normalized) offset: + # Accessing value by offset in sorted set: # - #[] (aliased as #slice): Returns the number or consecutive subset at a - # given offset or range of offsets. - # - #at: Returns the number at a given offset. - # - #find_index: Returns the given number's offset in the set + # given offset or range of offsets in the sorted set. + # - #at: Returns the number at a given offset in the sorted set. + # - #find_index: Returns the given number's offset in the sorted set. # # Set cardinality: # - #count (aliased as #size): Returns the count of numbers in the set. @@ -1083,10 +1083,10 @@ def has_duplicates? count_with_duplicates != count end - # Returns the index of +number+ in the set, or +nil+ if +number+ isn't in - # the set. + # Returns the (sorted and deduplicated) index of +number+ in the set, or + # +nil+ if +number+ isn't in the set. # - # Related: #[] + # Related: #[], #at def find_index(number) number = to_tuple_int number each_tuple_with_index do |min, max, idx_min| @@ -1120,8 +1120,11 @@ def reverse_each_tuple_with_index # :call-seq: at(index) -> integer or nil # - # Returns a number from +self+, without modifying the set. Behaves the - # same as #[], except that #at only allows a single integer argument. + # Returns the number at the given +index+ in the sorted set, without + # modifying the set. + # + # +index+ is interpreted the same as in #[], except that #at only allows a + # single integer argument. # # Related: #[], #slice def at(index) @@ -1146,17 +1149,18 @@ def at(index) # seqset[range] -> sequence set or nil # slice(range) -> sequence set or nil # - # Returns a number or a subset from +self+, without modifying the set. + # Returns a number or a subset from the _sorted_ set, without modifying + # the set. # # When an Integer argument +index+ is given, the number at offset +index+ - # is returned: + # in the sorted set is returned: # # set = Net::IMAP::SequenceSet["10:15,20:23,26"] # set[0] #=> 10 # set[5] #=> 15 # set[10] #=> 26 # - # If +index+ is negative, it counts relative to the end of +self+: + # If +index+ is negative, it counts relative to the end of the sorted set: # set = Net::IMAP::SequenceSet["10:15,20:23,26"] # set[-1] #=> 26 # set[-3] #=> 22 @@ -1168,13 +1172,14 @@ def at(index) # set[11] #=> nil # set[-12] #=> nil # - # The result is based on the normalized set—sorted and de-duplicated—not - # on the assigned value of #string. + # The result is based on the sorted and de-duplicated set, not on the + # ordered #entries in #string. # # set = Net::IMAP::SequenceSet["12,20:23,11:16,21"] # set[0] #=> 11 # set[-1] #=> 23 # + # Related: #at def [](index, length = nil) if length then slice_length(index, length) elsif index.is_a?(Range) then slice_range(index) From 5e69ce13abb3894a386f1edec8c81da1a619e684 Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 3 Feb 2025 09:46:04 -0500 Subject: [PATCH 22/34] =?UTF-8?q?=E2=9C=A8=20Add=20`SequenceSet#find=5Ford?= =?UTF-8?q?ered=5Findex`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the ordered entries version of `#find_index`. --- lib/net/imap/sequence_set.rb | 37 ++++++++++++++++++++++------- test/net/imap/test_sequence_set.rb | 38 ++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index 8f9f7d38..a8091c05 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -189,6 +189,10 @@ class IMAP # - #at: Returns the number at a given offset in the sorted set. # - #find_index: Returns the given number's offset in the sorted set. # + # Accessing value by offset in ordered entries + # - #find_ordered_index: Returns the index of the given number's first + # occurrence in entries. + # # Set cardinality: # - #count (aliased as #size): Returns the count of numbers in the set. # Duplicated numbers are not counted. @@ -1086,30 +1090,45 @@ def has_duplicates? # Returns the (sorted and deduplicated) index of +number+ in the set, or # +nil+ if +number+ isn't in the set. # - # Related: #[], #at + # Related: #[], #at, #find_ordered_index def find_index(number) number = to_tuple_int number - each_tuple_with_index do |min, max, idx_min| + each_tuple_with_index(@tuples) do |min, max, idx_min| number < min and return nil number <= max and return from_tuple_int(idx_min + (number - min)) end nil end + # Returns the first index of +number+ in the ordered #entries, or + # +nil+ if +number+ isn't in the set. + # + # Related: #find_index + def find_ordered_index(number) + number = to_tuple_int number + each_tuple_with_index(each_entry_tuple) do |min, max, idx_min| + if min <= number && number <= max + return from_tuple_int(idx_min + (number - min)) + end + end + nil + end + private - def each_tuple_with_index + def each_tuple_with_index(tuples) idx_min = 0 - @tuples.each do |min, max| - yield min, max, idx_min, (idx_max = idx_min + (max - min)) + tuples.each do |min, max| + idx_max = idx_min + (max - min) + yield min, max, idx_min, idx_max idx_min = idx_max + 1 end idx_min end - def reverse_each_tuple_with_index + def reverse_each_tuple_with_index(tuples) idx_max = -1 - @tuples.reverse_each do |min, max| + tuples.reverse_each do |min, max| yield min, max, (idx_min = idx_max - (max - min)), idx_max idx_max = idx_min - 1 end @@ -1130,11 +1149,11 @@ def reverse_each_tuple_with_index def at(index) index = Integer(index.to_int) if index.negative? - reverse_each_tuple_with_index do |min, max, idx_min, idx_max| + reverse_each_tuple_with_index(@tuples) do |min, max, idx_min, idx_max| idx_min <= index and return from_tuple_int(min + (index - idx_min)) end else - each_tuple_with_index do |min, _, idx_min, idx_max| + each_tuple_with_index(@tuples) do |min, _, idx_min, idx_max| index <= idx_max and return from_tuple_int(min + (index - idx_min)) end end diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index b6995305..58aefae5 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -251,6 +251,44 @@ def obj.to_sequence_set; 192_168.001_255 end assert_equal 2**32 - 1, SequenceSet.full.find_index(:*) end + test "#find_ordered_index" do + assert_equal 9, SequenceSet.full.find_ordered_index(10) + assert_equal 99, SequenceSet.full.find_ordered_index(100) + assert_equal 2**32 - 1, SequenceSet.full.find_ordered_index(:*) + assert_nil SequenceSet.empty.find_index(1) + set = SequenceSet["9,8,7,6,5,4,3,2,1"] + assert_equal 0, set.find_ordered_index(9) + assert_equal 1, set.find_ordered_index(8) + assert_equal 2, set.find_ordered_index(7) + assert_equal 3, set.find_ordered_index(6) + assert_equal 4, set.find_ordered_index(5) + assert_equal 5, set.find_ordered_index(4) + assert_equal 6, set.find_ordered_index(3) + assert_equal 7, set.find_ordered_index(2) + assert_equal 8, set.find_ordered_index(1) + assert_nil set.find_ordered_index(10) + set = SequenceSet["7:9,5:6"] + assert_equal 0, set.find_ordered_index(7) + assert_equal 1, set.find_ordered_index(8) + assert_equal 2, set.find_ordered_index(9) + assert_equal 3, set.find_ordered_index(5) + assert_equal 4, set.find_ordered_index(6) + assert_nil set.find_ordered_index(4) + set = SequenceSet["1000:1111,1:100"] + assert_equal 0, set.find_ordered_index(1000) + assert_equal 100, set.find_ordered_index(1100) + assert_equal 112, set.find_ordered_index(1) + assert_equal 121, set.find_ordered_index(10) + set = SequenceSet["1,1,1,1,51,50,4,11"] + assert_equal 0, set.find_ordered_index(1) + assert_equal 4, set.find_ordered_index(51) + assert_equal 5, set.find_ordered_index(50) + assert_equal 6, set.find_ordered_index(4) + assert_equal 7, set.find_ordered_index(11) + assert_equal 1, SequenceSet["1,*"].find_ordered_index(-1) + assert_equal 0, SequenceSet["*,1"].find_ordered_index(-1) + end + test "#limit" do set = SequenceSet["1:100,500"] assert_equal [1..99], set.limit(max: 99).ranges From 257ede0dfddd5c5496d5047c9fc673069ca9f1a2 Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 3 Feb 2025 18:49:38 -0500 Subject: [PATCH 23/34] =?UTF-8?q?=F0=9F=A5=85=20Re-raise=20`#starttls`=20e?= =?UTF-8?q?rror=20from=20receiver=20thread?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #394. When `start_tls_session` raises an exception, that's caught in the receiver thread, but not re-raised. Fortunately, `@sock` will now be a permanently broken SSLSocket, so I don't think this can lead to accidentally using an insecure connection. Even so, `#starttls` should disconnect the socket and re-raise the error immediately. Failing test case was provided by @rhenium in #394. Co-authored-by: Kazuki Yamaguchi --- lib/net/imap.rb | 10 +++++++++- test/net/imap/test_imap.rb | 13 ++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 7c8e0506..840e9c3e 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -1239,13 +1239,21 @@ def logout! # def starttls(**options) @ssl_ctx_params, @ssl_ctx = build_ssl_ctx(options) - send_command("STARTTLS") do |resp| + error = nil + ok = send_command("STARTTLS") do |resp| if resp.kind_of?(TaggedResponse) && resp.name == "OK" clear_cached_capabilities clear_responses start_tls_session end + rescue Exception => error + raise # note that the error backtrace is in the receiver_thread end + if error + disconnect + raise error + end + ok end # :call-seq: diff --git a/test/net/imap/test_imap.rb b/test/net/imap/test_imap.rb index 10a29c40..dd955b8d 100644 --- a/test/net/imap/test_imap.rb +++ b/test/net/imap/test_imap.rb @@ -113,17 +113,16 @@ def test_starttls_unknown_ca omit "This test is not working with Windows" if RUBY_PLATFORM =~ /mswin|mingw/ imap = nil - assert_raise(OpenSSL::SSL::SSLError) do - ex = nil - starttls_test do |port| - imap = Net::IMAP.new("localhost", port: port) + ex = nil + starttls_test do |port| + imap = Net::IMAP.new("localhost", port: port) + begin imap.starttls - imap rescue => ex - imap end - raise ex if ex + imap end + assert_kind_of(OpenSSL::SSL::SSLError, ex) assert_equal false, imap.tls_verified? assert_equal({}, imap.ssl_ctx_params) assert_equal(nil, imap.ssl_ctx.ca_file) From 608a9830c1fb5f278db5567cfde37942d2f9f7a0 Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 3 Feb 2025 15:16:41 -0500 Subject: [PATCH 24/34] =?UTF-8?q?=E2=9C=85=20Add=20missing=20specs=20for?= =?UTF-8?q?=20`SequenceSet#at`,=20`#[]`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some of the `#[negative index]` specs were using non-negative indexes. They've been updated to use negative indexes. Previously, `#at` was indirectly tested only by the `SequenceSet#[]` test and the randomized "compare to reference Set" tests. The new tests specifically explicitly focus on `#[]`. --- test/net/imap/test_sequence_set.rb | 45 ++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index 58aefae5..4ff6753a 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -167,6 +167,21 @@ def obj.to_sequence_set; 192_168.001_255 end assert_raise DataFormatError do SequenceSet.try_convert(obj) end end + test "#at(non-negative index)" do + assert_nil SequenceSet.empty.at(0) + assert_equal 1, SequenceSet[1..].at(0) + assert_equal 1, SequenceSet.full.at(0) + assert_equal 111, SequenceSet.full.at(110) + assert_equal 4, SequenceSet[2,4,6,8].at(1) + assert_equal 8, SequenceSet[2,4,6,8].at(3) + assert_equal 6, SequenceSet[4..6].at(2) + assert_nil SequenceSet[4..6].at(3) + assert_equal 205, SequenceSet["101:110,201:210,301:310"].at(14) + assert_equal 310, SequenceSet["101:110,201:210,301:310"].at(29) + assert_nil SequenceSet["101:110,201:210,301:310"].at(44) + assert_equal :*, SequenceSet["1:10,*"].at(10) + end + test "#[non-negative index]" do assert_nil SequenceSet.empty[0] assert_equal 1, SequenceSet[1..][0] @@ -182,18 +197,32 @@ def obj.to_sequence_set; 192_168.001_255 end assert_equal :*, SequenceSet["1:10,*"][10] end + test "#at(negative index)" do + assert_nil SequenceSet.empty.at(-1) + assert_equal :*, SequenceSet[1..].at(-1) + assert_equal 1, SequenceSet.full.at(-(2**32)) + assert_equal 111, SequenceSet[1..111].at(-1) + assert_equal 6, SequenceSet[2,4,6,8].at(-2) + assert_equal 2, SequenceSet[2,4,6,8].at(-4) + assert_equal 4, SequenceSet[4..6].at(-3) + assert_nil SequenceSet[4..6].at(-4) + assert_equal 207, SequenceSet["101:110,201:210,301:310"].at(-14) + assert_equal 102, SequenceSet["101:110,201:210,301:310"].at(-29) + assert_nil SequenceSet["101:110,201:210,301:310"].at(-44) + end + test "#[negative index]" do - assert_nil SequenceSet.empty[0] + assert_nil SequenceSet.empty[-1] assert_equal :*, SequenceSet[1..][-1] assert_equal 1, SequenceSet.full[-(2**32)] assert_equal 111, SequenceSet[1..111][-1] - assert_equal 4, SequenceSet[2,4,6,8][1] - assert_equal 8, SequenceSet[2,4,6,8][3] - assert_equal 6, SequenceSet[4..6][2] - assert_nil SequenceSet[4..6][3] - assert_equal 205, SequenceSet["101:110,201:210,301:310"][14] - assert_equal 310, SequenceSet["101:110,201:210,301:310"][29] - assert_nil SequenceSet["101:110,201:210,301:310"][44] + assert_equal 6, SequenceSet[2,4,6,8][-2] + assert_equal 2, SequenceSet[2,4,6,8][-4] + assert_equal 4, SequenceSet[4..6][-3] + assert_nil SequenceSet[4..6][-4] + assert_equal 207, SequenceSet["101:110,201:210,301:310"][-14] + assert_equal 102, SequenceSet["101:110,201:210,301:310"][-29] + assert_nil SequenceSet["101:110,201:210,301:310"][-44] end test "#[start, length]" do From 54de6575c43c206d1aa8875f86a7a54914aec35f Mon Sep 17 00:00:00 2001 From: nick evans Date: Mon, 3 Feb 2025 09:51:33 -0500 Subject: [PATCH 25/34] =?UTF-8?q?=E2=9C=A8=20Add=20`SequenceSet#ordered=5F?= =?UTF-8?q?at`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the ordered entries variation of `#at`. --- lib/net/imap/sequence_set.rb | 24 +++++++++++++++++++--- test/net/imap/test_sequence_set.rb | 32 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/lib/net/imap/sequence_set.rb b/lib/net/imap/sequence_set.rb index a8091c05..02cd150d 100644 --- a/lib/net/imap/sequence_set.rb +++ b/lib/net/imap/sequence_set.rb @@ -190,6 +190,7 @@ class IMAP # - #find_index: Returns the given number's offset in the sorted set. # # Accessing value by offset in ordered entries + # - #ordered_at: Returns the number at a given offset in the ordered entries. # - #find_ordered_index: Returns the index of the given number's first # occurrence in entries. # @@ -1145,15 +1146,32 @@ def reverse_each_tuple_with_index(tuples) # +index+ is interpreted the same as in #[], except that #at only allows a # single integer argument. # - # Related: #[], #slice + # Related: #[], #slice, #ordered_at def at(index) + lookup_number_by_tuple_index(tuples, index) + end + + # :call-seq: ordered_at(index) -> integer or nil + # + # Returns the number at the given +index+ in the ordered #entries, without + # modifying the set. + # + # +index+ is interpreted the same as in #at (and #[]), except that + # #ordered_at applies to the ordered #entries, not the sorted set. + # + # Related: #[], #slice, #ordered_at + def ordered_at(index) + lookup_number_by_tuple_index(each_entry_tuple, index) + end + + private def lookup_number_by_tuple_index(tuples, index) index = Integer(index.to_int) if index.negative? - reverse_each_tuple_with_index(@tuples) do |min, max, idx_min, idx_max| + reverse_each_tuple_with_index(tuples) do |min, max, idx_min, idx_max| idx_min <= index and return from_tuple_int(min + (index - idx_min)) end else - each_tuple_with_index(@tuples) do |min, _, idx_min, idx_max| + each_tuple_with_index(tuples) do |min, _, idx_min, idx_max| index <= idx_max and return from_tuple_int(min + (index - idx_min)) end end diff --git a/test/net/imap/test_sequence_set.rb b/test/net/imap/test_sequence_set.rb index 4ff6753a..aad844ac 100644 --- a/test/net/imap/test_sequence_set.rb +++ b/test/net/imap/test_sequence_set.rb @@ -225,6 +225,38 @@ def obj.to_sequence_set; 192_168.001_255 end assert_nil SequenceSet["101:110,201:210,301:310"][-44] end + test "#ordered_at(non-negative index)" do + assert_nil SequenceSet.empty.ordered_at(0) + assert_equal 1, SequenceSet.full.ordered_at(0) + assert_equal 111, SequenceSet.full.ordered_at(110) + assert_equal 1, SequenceSet["1:*"].ordered_at(0) + assert_equal :*, SequenceSet["*,1"].ordered_at(0) + assert_equal 4, SequenceSet["6,4,8,2"].ordered_at(1) + assert_equal 2, SequenceSet["6,4,8,2"].ordered_at(3) + assert_equal 6, SequenceSet["9:11,4:6,1:3"].ordered_at(5) + assert_nil SequenceSet["9:11,4:6,1:3"].ordered_at(9) + assert_equal 105, SequenceSet["201:210,101:110,301:310"].ordered_at(14) + assert_equal 310, SequenceSet["201:210,101:110,301:310"].ordered_at(29) + assert_nil SequenceSet["201:210,101:110,301:310"].ordered_at(30) + assert_equal :*, SequenceSet["1:10,*"].ordered_at(10) + end + + test "#ordered_at(negative index)" do + assert_nil SequenceSet.empty.ordered_at(-1) + assert_equal :*, SequenceSet["1:*"].ordered_at(-1) + assert_equal 1, SequenceSet.full.ordered_at(-(2**32)) + assert_equal :*, SequenceSet["*,1"].ordered_at(0) + assert_equal 8, SequenceSet["6,4,8,2"].ordered_at(-2) + assert_equal 6, SequenceSet["6,4,8,2"].ordered_at(-4) + assert_equal 4, SequenceSet["9:11,4:6,1:3"].ordered_at(-6) + assert_equal 10, SequenceSet["9:11,4:6,1:3"].ordered_at(-8) + assert_nil SequenceSet["9:11,4:6,1:3"].ordered_at(-12) + assert_equal 107, SequenceSet["201:210,101:110,301:310"].ordered_at(-14) + assert_equal 201, SequenceSet["201:210,101:110,301:310"].ordered_at(-30) + assert_nil SequenceSet["201:210,101:110,301:310"].ordered_at(-31) + assert_equal :*, SequenceSet["1:10,*"].ordered_at(10) + end + test "#[start, length]" do assert_equal SequenceSet[10..99], SequenceSet.full[9, 90] assert_equal 90, SequenceSet.full[9, 90].count From 5617a036e7f9bea496314b06ad61d68c383cf7f1 Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 5 Feb 2025 16:31:36 -0500 Subject: [PATCH 26/34] =?UTF-8?q?=F0=9F=93=9A=20Document=20COPYUID=20in=20?= =?UTF-8?q?tagged=20vs=20untagged=20responses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/uidplus_data.rb | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/net/imap/uidplus_data.rb b/lib/net/imap/uidplus_data.rb index 2c478e2c..687e34c7 100644 --- a/lib/net/imap/uidplus_data.rb +++ b/lib/net/imap/uidplus_data.rb @@ -6,12 +6,20 @@ class IMAP < Protocol # UIDPlusData represents the ResponseCode#data that accompanies the # +APPENDUID+ and +COPYUID+ {response codes}[rdoc-ref:ResponseCode]. # - # A server that supports +UIDPLUS+ should send a UIDPlusData object inside - # every TaggedResponse returned by the append[rdoc-ref:Net::IMAP#append], - # copy[rdoc-ref:Net::IMAP#copy], move[rdoc-ref:Net::IMAP#move], {uid - # copy}[rdoc-ref:Net::IMAP#uid_copy], and {uid - # move}[rdoc-ref:Net::IMAP#uid_move] commands---unless the destination - # mailbox reports +UIDNOTSTICKY+. + # A server that supports +UIDPLUS+ should send UIDPlusData in response to + # the append[rdoc-ref:Net::IMAP#append], copy[rdoc-ref:Net::IMAP#copy], + # move[rdoc-ref:Net::IMAP#move], {uid copy}[rdoc-ref:Net::IMAP#uid_copy], + # and {uid move}[rdoc-ref:Net::IMAP#uid_move] commands---unless the + # destination mailbox reports +UIDNOTSTICKY+. + # + # Note that append[rdoc-ref:Net::IMAP#append], copy[rdoc-ref:Net::IMAP#copy] + # and {uid_copy}[rdoc-ref:Net::IMAP#uid_copy] return UIDPlusData in their + # TaggedResponse. But move[rdoc-ref:Net::IMAP#copy] and + # {uid_move}[rdoc-ref:Net::IMAP#uid_move] _should_ send UIDPlusData in an + # UntaggedResponse response before sending their TaggedResponse. However + # some servers do send UIDPlusData in the TaggedResponse for +MOVE+ + # commands---this complies with the older +UIDPLUS+ specification but is + # discouraged by the +MOVE+ extension and disallowed by +IMAP4rev2+. # # == Required capability # Requires either +UIDPLUS+ [RFC4315[https://www.rfc-editor.org/rfc/rfc4315]] From 85d0aa264daa682106c73bd4a33d047888b3eee7 Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 5 Feb 2025 16:24:54 -0500 Subject: [PATCH 27/34] =?UTF-8?q?=F0=9F=9A=9A=20Rename=20UIDPLUS=20test=20?= =?UTF-8?q?file=20for=20consistency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/net/imap/{test_uid_plus_data.rb => test_uidplus_data.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/net/imap/{test_uid_plus_data.rb => test_uidplus_data.rb} (100%) diff --git a/test/net/imap/test_uid_plus_data.rb b/test/net/imap/test_uidplus_data.rb similarity index 100% rename from test/net/imap/test_uid_plus_data.rb rename to test/net/imap/test_uidplus_data.rb From 01bb49f4ae3220a695e21314ba4d92a84fe64b35 Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 5 Feb 2025 16:25:11 -0500 Subject: [PATCH 28/34] =?UTF-8?q?=E2=9C=A8=20Add=20AppendUIDData=20(to=20r?= =?UTF-8?q?eplace=20UIDPlusData)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/response_data.rb | 1 + lib/net/imap/uidplus_data.rb | 39 ++++++++++++++++++++++++++++++ test/net/imap/test_uidplus_data.rb | 36 +++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/lib/net/imap/response_data.rb b/lib/net/imap/response_data.rb index 40586ebd..37f06c22 100644 --- a/lib/net/imap/response_data.rb +++ b/lib/net/imap/response_data.rb @@ -8,6 +8,7 @@ class IMAP < Protocol autoload :SearchResult, "#{__dir__}/search_result" autoload :SequenceSet, "#{__dir__}/sequence_set" autoload :UIDPlusData, "#{__dir__}/uidplus_data" + autoload :AppendUIDData, "#{__dir__}/uidplus_data" autoload :VanishedData, "#{__dir__}/vanished_data" # Net::IMAP::ContinuationRequest represents command continuation requests. diff --git a/lib/net/imap/uidplus_data.rb b/lib/net/imap/uidplus_data.rb index 687e34c7..dae0bf01 100644 --- a/lib/net/imap/uidplus_data.rb +++ b/lib/net/imap/uidplus_data.rb @@ -60,5 +60,44 @@ def uid_mapping end end + # AppendUIDData represents the ResponseCode#data that accompanies the + # +APPENDUID+ {response code}[rdoc-ref:ResponseCode]. + # + # A server that supports +UIDPLUS+ (or +IMAP4rev2+) should send + # AppendUIDData inside every TaggedResponse returned by the + # append[rdoc-ref:Net::IMAP#append] command---unless the target mailbox + # reports +UIDNOTSTICKY+. + # + # == Required capability + # Requires either +UIDPLUS+ [RFC4315[https://www.rfc-editor.org/rfc/rfc4315]] + # or +IMAP4rev2+ capability. + class AppendUIDData < Data.define(:uidvalidity, :assigned_uids) + def initialize(uidvalidity:, assigned_uids:) + uidvalidity = Integer(uidvalidity) + assigned_uids = SequenceSet[assigned_uids] + NumValidator.ensure_nz_number(uidvalidity) + if assigned_uids.include_star? + raise DataFormatError, "uid-set cannot contain '*'" + end + super + end + + ## + # attr_reader: uidvalidity + # :call-seq: uidvalidity -> nonzero uint32 + # + # The UIDVALIDITY of the destination mailbox. + + ## + # attr_reader: assigned_uids + # + # A SequenceSet with the newly assigned UIDs of the appended messages. + + # Returns the number of messages that have been appended. + def size + assigned_uids.count_with_duplicates + end + end + end end diff --git a/test/net/imap/test_uidplus_data.rb b/test/net/imap/test_uidplus_data.rb index 210a000e..0d693ae9 100644 --- a/test/net/imap/test_uidplus_data.rb +++ b/test/net/imap/test_uidplus_data.rb @@ -44,3 +44,39 @@ class TestUIDPlusData < Test::Unit::TestCase end end + +class TestAppendUIDData < Test::Unit::TestCase + # alias for convenience + AppendUIDData = Net::IMAP::AppendUIDData + SequenceSet = Net::IMAP::SequenceSet + DataFormatError = Net::IMAP::DataFormatError + UINT32_MAX = 2**32 - 1 + + test "#uidvalidity must be valid nz-number" do + assert_equal 1, AppendUIDData.new(1, 99).uidvalidity + assert_equal UINT32_MAX, AppendUIDData.new(UINT32_MAX, 1).uidvalidity + assert_raise DataFormatError do AppendUIDData.new(0, 1) end + assert_raise DataFormatError do AppendUIDData.new(2**32, 1) end + end + + test "#assigned_uids must be a valid uid-set" do + assert_equal SequenceSet[1], AppendUIDData.new(99, "1").assigned_uids + assert_equal SequenceSet[1..9], AppendUIDData.new(1, "1:9").assigned_uids + assert_equal(SequenceSet[UINT32_MAX], + AppendUIDData.new(1, UINT32_MAX.to_s).assigned_uids) + assert_raise DataFormatError do AppendUIDData.new(1, 0) end + assert_raise DataFormatError do AppendUIDData.new(1, "*") end + assert_raise DataFormatError do AppendUIDData.new(1, "1:*") end + end + + test "#size returns the number of UIDs" do + assert_equal(10, AppendUIDData.new(1, "1:10").size) + assert_equal(4_000_000_000, AppendUIDData.new(1, 1..4_000_000_000).size) + end + + test "#assigned_uids is converted to SequenceSet" do + assert_equal SequenceSet[1], AppendUIDData.new(99, "1").assigned_uids + assert_equal SequenceSet[1..4], AppendUIDData.new(1, [1, 2, 3, 4]).assigned_uids + end + +end From bcb261d12e9911eaf89d35db314c626501c92b72 Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 5 Feb 2025 16:25:36 -0500 Subject: [PATCH 29/34] =?UTF-8?q?=E2=9C=A8=20Add=20CopyUIDData=20(to=20rep?= =?UTF-8?q?lace=20UIDPlusData)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/response_data.rb | 1 + lib/net/imap/uidplus_data.rb | 127 ++++++++++++++++++++++++ test/net/imap/test_uidplus_data.rb | 150 +++++++++++++++++++++++++++++ 3 files changed, 278 insertions(+) diff --git a/lib/net/imap/response_data.rb b/lib/net/imap/response_data.rb index 37f06c22..d862deaa 100644 --- a/lib/net/imap/response_data.rb +++ b/lib/net/imap/response_data.rb @@ -9,6 +9,7 @@ class IMAP < Protocol autoload :SequenceSet, "#{__dir__}/sequence_set" autoload :UIDPlusData, "#{__dir__}/uidplus_data" autoload :AppendUIDData, "#{__dir__}/uidplus_data" + autoload :CopyUIDData, "#{__dir__}/uidplus_data" autoload :VanishedData, "#{__dir__}/vanished_data" # Net::IMAP::ContinuationRequest represents command continuation requests. diff --git a/lib/net/imap/uidplus_data.rb b/lib/net/imap/uidplus_data.rb index dae0bf01..f937d53d 100644 --- a/lib/net/imap/uidplus_data.rb +++ b/lib/net/imap/uidplus_data.rb @@ -99,5 +99,132 @@ def size end end + # CopyUIDData represents the ResponseCode#data that accompanies the + # +COPYUID+ {response code}[rdoc-ref:ResponseCode]. + # + # A server that supports +UIDPLUS+ (or +IMAP4rev2+) should send CopyUIDData + # in response to + # copy[rdoc-ref:Net::IMAP#copy], {uid_copy}[rdoc-ref:Net::IMAP#uid_copy], + # move[rdoc-ref:Net::IMAP#copy], and {uid_move}[rdoc-ref:Net::IMAP#uid_move] + # commands---unless the destination mailbox reports +UIDNOTSTICKY+. + # + # Note that copy[rdoc-ref:Net::IMAP#copy] and + # {uid_copy}[rdoc-ref:Net::IMAP#uid_copy] return CopyUIDData in their + # TaggedResponse. But move[rdoc-ref:Net::IMAP#copy] and + # {uid_move}[rdoc-ref:Net::IMAP#uid_move] _should_ send CopyUIDData in an + # UntaggedResponse response before sending their TaggedResponse. However + # some servers do send CopyUIDData in the TaggedResponse for +MOVE+ + # commands---this complies with the older +UIDPLUS+ specification but is + # discouraged by the +MOVE+ extension and disallowed by +IMAP4rev2+. + # + # == Required capability + # Requires either +UIDPLUS+ [RFC4315[https://www.rfc-editor.org/rfc/rfc4315]] + # or +IMAP4rev2+ capability. + class CopyUIDData < Data.define(:uidvalidity, :source_uids, :assigned_uids) + def initialize(uidvalidity:, source_uids:, assigned_uids:) + uidvalidity = Integer(uidvalidity) + source_uids = SequenceSet[source_uids] + assigned_uids = SequenceSet[assigned_uids] + NumValidator.ensure_nz_number(uidvalidity) + if source_uids.include_star? || assigned_uids.include_star? + raise DataFormatError, "uid-set cannot contain '*'" + elsif source_uids.count_with_duplicates != assigned_uids.count_with_duplicates + raise DataFormatError, "mismatched uid-set sizes for %s and %s" % [ + source_uids, assigned_uids + ] + end + super + end + + ## + # attr_reader: uidvalidity + # + # The +UIDVALIDITY+ of the destination mailbox (a nonzero unsigned 32 bit + # integer). + + ## + # attr_reader: source_uids + # + # A SequenceSet with the original UIDs of the copied or moved messages. + + ## + # attr_reader: assigned_uids + # + # A SequenceSet with the newly assigned UIDs of the copied or moved + # messages. + + # Returns the number of messages that have been copied or moved. + # source_uids and the assigned_uids will both the same number of UIDs. + def size + assigned_uids.count_with_duplicates + end + + # :call-seq: + # assigned_uid_for(source_uid) -> uid + # self[source_uid] -> uid + # + # Returns the UID in the destination mailbox for the message that was + # copied from +source_uid+ in the source mailbox. + # + # This is the reverse of #source_uid_for. + # + # Related: source_uid_for, each_uid_pair, uid_mapping + def assigned_uid_for(source_uid) + idx = source_uids.find_ordered_index(source_uid) and + assigned_uids.ordered_at(idx) + end + alias :[] :assigned_uid_for + + # :call-seq: + # source_uid_for(assigned_uid) -> uid + # + # Returns the UID in the source mailbox for the message that was copied to + # +assigned_uid+ in the source mailbox. + # + # This is the reverse of #assigned_uid_for. + # + # Related: assigned_uid_for, each_uid_pair, uid_mapping + def source_uid_for(assigned_uid) + idx = assigned_uids.find_ordered_index(assigned_uid) and + source_uids.ordered_at(idx) + end + + # Yields a pair of UIDs for each copied message. The first is the + # message's UID in the source mailbox and the second is the UID in the + # destination mailbox. + # + # Returns an enumerator when no block is given. + # + # Please note the warning on uid_mapping before calling methods like + # +to_h+ or +to_a+ on the returned enumerator. + # + # Related: uid_mapping, assigned_uid_for, source_uid_for + def each_uid_pair + return enum_for(__method__) unless block_given? + source_uids.each_ordered_number.lazy + .zip(assigned_uids.each_ordered_number.lazy) do + |source_uid, assigned_uid| + yield source_uid, assigned_uid + end + end + alias each_pair each_uid_pair + alias each each_uid_pair + + # :call-seq: uid_mapping -> hash + # + # Returns a hash mapping each source UID to the newly assigned destination + # UID. + # + # *Warning:* The hash that is created may consume _much_ more + # memory than the data used to create it. When handling responses from an + # untrusted server, check #size before calling this method. + # + # Related: each_uid_pair, assigned_uid_for, source_uid_for + def uid_mapping + each_uid_pair.to_h + end + + end + end end diff --git a/test/net/imap/test_uidplus_data.rb b/test/net/imap/test_uidplus_data.rb index 0d693ae9..0088c346 100644 --- a/test/net/imap/test_uidplus_data.rb +++ b/test/net/imap/test_uidplus_data.rb @@ -80,3 +80,153 @@ class TestAppendUIDData < Test::Unit::TestCase end end + +class TestCopyUIDData < Test::Unit::TestCase + # alias for convenience + CopyUIDData = Net::IMAP::CopyUIDData + SequenceSet = Net::IMAP::SequenceSet + DataFormatError = Net::IMAP::DataFormatError + UINT32_MAX = 2**32 - 1 + + test "#uidvalidity must be valid nz-number" do + assert_equal 1, CopyUIDData.new(1, 99, 99).uidvalidity + assert_equal UINT32_MAX, CopyUIDData.new(UINT32_MAX, 1, 1).uidvalidity + assert_raise DataFormatError do CopyUIDData.new(0, 1, 1) end + assert_raise DataFormatError do CopyUIDData.new(2**32, 1, 1) end + end + + test "#source_uids must be valid uid-set" do + assert_equal SequenceSet[1], CopyUIDData.new(99, "1", 99).source_uids + assert_equal SequenceSet[5..8], CopyUIDData.new(1, 5..8, 1..4).source_uids + assert_equal(SequenceSet[UINT32_MAX], + CopyUIDData.new(1, UINT32_MAX.to_s, 1).source_uids) + assert_raise DataFormatError do CopyUIDData.new(99, nil, 99) end + assert_raise DataFormatError do CopyUIDData.new(1, 0, 1) end + assert_raise DataFormatError do CopyUIDData.new(1, "*", 1) end + end + + test "#assigned_uids must be a valid uid-set" do + assert_equal SequenceSet[1], CopyUIDData.new(99, 1, "1").assigned_uids + assert_equal SequenceSet[1..9], CopyUIDData.new(1, 1..9, "1:9").assigned_uids + assert_equal(SequenceSet[UINT32_MAX], + CopyUIDData.new(1, 1, UINT32_MAX.to_s).assigned_uids) + assert_raise DataFormatError do CopyUIDData.new(1, 1, 0) end + assert_raise DataFormatError do CopyUIDData.new(1, 1, "*") end + assert_raise DataFormatError do CopyUIDData.new(1, 1, "1:*") end + end + + test "#size returns the number of UIDs" do + assert_equal(10, CopyUIDData.new(1, "9,8,7,6,1:5,10", "1:10").size) + assert_equal(4_000_000_000, + CopyUIDData.new( + 1, "2000000000:4000000000,1:1999999999", 1..4_000_000_000 + ).size) + end + + test "#source_uids and #assigned_uids must be same size" do + assert_raise DataFormatError do CopyUIDData.new(1, 1..5, 1) end + assert_raise DataFormatError do CopyUIDData.new(1, 1, 1..5) end + end + + test "#source_uids is converted to SequenceSet" do + assert_equal SequenceSet[1], CopyUIDData.new(99, "1", 99).source_uids + assert_equal SequenceSet[5, 6, 7, 8], CopyUIDData.new(1, 5..8, 1..4).source_uids + end + + test "#assigned_uids is converted to SequenceSet" do + assert_equal SequenceSet[1], CopyUIDData.new(99, 1, "1").assigned_uids + assert_equal SequenceSet[1, 2, 3, 4], CopyUIDData.new(1, "1:4", 1..4).assigned_uids + end + + test "#uid_mapping maps source_uids to assigned_uids" do + uidplus = CopyUIDData.new(9999, "20:19,500:495", "92:97,101:100") + assert_equal( + { + 19 => 92, + 20 => 93, + 495 => 94, + 496 => 95, + 497 => 96, + 498 => 97, + 499 => 100, + 500 => 101, + }, + uidplus.uid_mapping + ) + end + + test "#uid_mapping for with source_uids in unsorted order" do + uidplus = CopyUIDData.new(1, "495:500,20:19", "92:97,101:100") + assert_equal( + { + 495 => 92, + 496 => 93, + 497 => 94, + 498 => 95, + 499 => 96, + 500 => 97, + 19 => 100, + 20 => 101, + }, + uidplus.uid_mapping + ) + end + + test "#assigned_uid_for(source_uid)" do + uidplus = CopyUIDData.new(1, "495:500,20:19", "92:97,101:100") + assert_equal 92, uidplus.assigned_uid_for(495) + assert_equal 93, uidplus.assigned_uid_for(496) + assert_equal 94, uidplus.assigned_uid_for(497) + assert_equal 95, uidplus.assigned_uid_for(498) + assert_equal 96, uidplus.assigned_uid_for(499) + assert_equal 97, uidplus.assigned_uid_for(500) + assert_equal 100, uidplus.assigned_uid_for( 19) + assert_equal 101, uidplus.assigned_uid_for( 20) + end + + test "#[](source_uid)" do + uidplus = CopyUIDData.new(1, "495:500,20:19", "92:97,101:100") + assert_equal 92, uidplus[495] + assert_equal 93, uidplus[496] + assert_equal 94, uidplus[497] + assert_equal 95, uidplus[498] + assert_equal 96, uidplus[499] + assert_equal 97, uidplus[500] + assert_equal 100, uidplus[ 19] + assert_equal 101, uidplus[ 20] + end + + test "#source_uid_for(assigned_uid)" do + uidplus = CopyUIDData.new(1, "495:500,20:19", "92:97,101:100") + assert_equal 495, uidplus.source_uid_for( 92) + assert_equal 496, uidplus.source_uid_for( 93) + assert_equal 497, uidplus.source_uid_for( 94) + assert_equal 498, uidplus.source_uid_for( 95) + assert_equal 499, uidplus.source_uid_for( 96) + assert_equal 500, uidplus.source_uid_for( 97) + assert_equal 19, uidplus.source_uid_for(100) + assert_equal 20, uidplus.source_uid_for(101) + end + + test "#each_uid_pair" do + uidplus = CopyUIDData.new(1, "495:500,20:19", "92:97,101:100") + expected = { + 495 => 92, + 496 => 93, + 497 => 94, + 498 => 95, + 499 => 96, + 500 => 97, + 19 => 100, + 20 => 101, + } + actual = {} + uidplus.each_uid_pair do |src, dst| actual[src] = dst end + assert_equal expected, actual + assert_equal expected, uidplus.each_uid_pair.to_h + assert_equal expected.to_a, uidplus.each_uid_pair.to_a + assert_equal expected, uidplus.each_pair.to_h + assert_equal expected, uidplus.each.to_h + end + +end From 60f577690d80dd2593edaeb1d09b7681bedac368 Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 5 Feb 2025 16:25:52 -0500 Subject: [PATCH 30/34] =?UTF-8?q?=F0=9F=94=A7=F0=9F=97=91=EF=B8=8F=20Depre?= =?UTF-8?q?cate=20UIDPlusData,=20with=20config=20to=20upgrade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This config attribute causes the parser to use the new AppendUIDData and CopyUIDData classes instead of CopyUIDData. AppendUIDData and CopyUIDData are _mostly_ backward-compatible with UIDPlusData. Most applications should be able to upgrade with no changes. UIDPlusData will be removed in +v0.6+. --- lib/net/imap/config.rb | 29 +++++++++++++++++ lib/net/imap/response_parser.rb | 12 ++++--- lib/net/imap/uidplus_data.rb | 14 ++++++++ test/net/imap/test_imap_response_parser.rb | 37 ++++++++++++++++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/lib/net/imap/config.rb b/lib/net/imap/config.rb index edeea30e..d5b0975a 100644 --- a/lib/net/imap/config.rb +++ b/lib/net/imap/config.rb @@ -287,6 +287,32 @@ def self.[](config) # # Alias for responses_without_block + # Whether ResponseParser should use the deprecated UIDPlusData or + # CopyUIDData for +COPYUID+ response codes, and UIDPlusData or + # AppendUIDData for +APPENDUID+ response codes. + # + # AppendUIDData and CopyUIDData are _mostly_ backward-compatible with + # UIDPlusData. Most applications should be able to upgrade with little + # or no changes. + # + # (Parser support for +UIDPLUS+ added in +v0.3.2+.) + # + # (Config option added in +v0.4.19+ and +v0.5.6+.) + # + # UIDPlusData will be removed in +v0.6+ and this config setting will + # be ignored. + # + # ==== Valid options + # + # [+true+ (original default)] + # ResponseParser only uses UIDPlusData. + # + # [+false+ (planned default for +v0.6+)] + # ResponseParser _only_ uses AppendUIDData and CopyUIDData. + attr_accessor :parser_use_deprecated_uidplus_data, type: [ + true, false + ] + # Creates a new config object and initialize its attribute with +attrs+. # # If +parent+ is not given, the global config is used by default. @@ -367,6 +393,7 @@ def defaults_hash sasl_ir: true, enforce_logindisabled: true, responses_without_block: :warn, + parser_use_deprecated_uidplus_data: true, ).freeze @global = default.new @@ -378,6 +405,7 @@ def defaults_hash sasl_ir: false, responses_without_block: :silence_deprecation_warning, enforce_logindisabled: false, + parser_use_deprecated_uidplus_data: true, ).freeze version_defaults[0.0] = Config[0] version_defaults[0.1] = Config[0] @@ -392,6 +420,7 @@ def defaults_hash version_defaults[0.6] = Config[0.5].dup.update( responses_without_block: :frozen_dup, + parser_use_deprecated_uidplus_data: false, ).freeze version_defaults[:next] = Config[0.6] version_defaults[:future] = Config[:next] diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index 03f135ec..71c08e7b 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -2001,11 +2001,10 @@ def charset__list # # n.b, uniqueid ⊂ uid-set. To avoid inconsistent return types, we always # match uid_set even if that returns a single-member array. - # def resp_code_apnd__data validity = number; SP! dst_uids = uid_set # uniqueid ⊂ uid-set - UIDPlus(validity, nil, dst_uids) + AppendUID(validity, dst_uids) end # already matched: "COPYUID" @@ -2015,10 +2014,15 @@ def resp_code_copy__data validity = number; SP! src_uids = uid_set; SP! dst_uids = uid_set - UIDPlus(validity, src_uids, dst_uids) + CopyUID(validity, src_uids, dst_uids) end - def UIDPlus(validity, src_uids, dst_uids) + def AppendUID(...) DeprecatedUIDPlus(...) || AppendUIDData.new(...) end + def CopyUID(...) DeprecatedUIDPlus(...) || CopyUIDData.new(...) end + + # TODO: remove this code in the v0.6.0 release + def DeprecatedUIDPlus(validity, src_uids = nil, dst_uids) + return unless config.parser_use_deprecated_uidplus_data src_uids &&= src_uids.each_ordered_number.to_a dst_uids = dst_uids.each_ordered_number.to_a UIDPlusData.new(validity, src_uids, dst_uids) diff --git a/lib/net/imap/uidplus_data.rb b/lib/net/imap/uidplus_data.rb index f937d53d..0e593636 100644 --- a/lib/net/imap/uidplus_data.rb +++ b/lib/net/imap/uidplus_data.rb @@ -3,6 +3,10 @@ module Net class IMAP < Protocol + # *NOTE:* UIDPlusData is deprecated and will be removed in the +0.6.0+ + # release. To use AppendUIDData and CopyUIDData before +0.6.0+, set + # Config#parser_use_deprecated_uidplus_data to +false+. + # # UIDPlusData represents the ResponseCode#data that accompanies the # +APPENDUID+ and +COPYUID+ {response codes}[rdoc-ref:ResponseCode]. # @@ -60,6 +64,11 @@ def uid_mapping end end + # >>> + # *NOTE:* AppendUIDData will replace UIDPlusData for +APPENDUID+ in the + # +0.6.0+ release. To use AppendUIDData before +0.6.0+, set + # Config#parser_use_deprecated_uidplus_data to +false+. + # # AppendUIDData represents the ResponseCode#data that accompanies the # +APPENDUID+ {response code}[rdoc-ref:ResponseCode]. # @@ -99,6 +108,11 @@ def size end end + # >>> + # *NOTE:* CopyUIDData will replace UIDPlusData for +COPYUID+ in the + # +0.6.0+ release. To use CopyUIDData before +0.6.0+, set + # Config#parser_use_deprecated_uidplus_data to +false+. + # # CopyUIDData represents the ResponseCode#data that accompanies the # +COPYUID+ {response code}[rdoc-ref:ResponseCode]. # diff --git a/test/net/imap/test_imap_response_parser.rb b/test/net/imap/test_imap_response_parser.rb index b24c63ff..7a70c02b 100644 --- a/test/net/imap/test_imap_response_parser.rb +++ b/test/net/imap/test_imap_response_parser.rb @@ -211,6 +211,24 @@ def test_fetch_binary_and_binary_size end end + test "APPENDUID with parser_use_deprecated_uidplus_data = true" do + parser = Net::IMAP::ResponseParser.new(config: { + parser_use_deprecated_uidplus_data: true, + }) + response = parser.parse("A004 OK [APPENDUID 1 101:200] Done\r\n") + uidplus = response.data.code.data + assert_instance_of Net::IMAP::UIDPlusData, uidplus + assert_equal 100, uidplus.assigned_uids.size + end + + test "APPENDUID with parser_use_deprecated_uidplus_data = false" do + parser = Net::IMAP::ResponseParser.new(config: { + parser_use_deprecated_uidplus_data: false, + }) + response = parser.parse("A004 OK [APPENDUID 1 10] Done\r\n") + assert_instance_of Net::IMAP::AppendUIDData, response.data.code.data + end + test "COPYUID with backwards ranges" do parser = Net::IMAP::ResponseParser.new response = parser.parse( @@ -241,4 +259,23 @@ def test_fetch_binary_and_binary_size end end + test "COPYUID with parser_use_deprecated_uidplus_data = true" do + parser = Net::IMAP::ResponseParser.new(config: { + parser_use_deprecated_uidplus_data: true, + }) + response = parser.parse("A004 OK [copyUID 1 101:200 1:100] Done\r\n") + uidplus = response.data.code.data + assert_instance_of Net::IMAP::UIDPlusData, uidplus + assert_equal 100, uidplus.assigned_uids.size + assert_equal 100, uidplus.source_uids.size + end + + test "COPYUID with parser_use_deprecated_uidplus_data = false" do + parser = Net::IMAP::ResponseParser.new(config: { + parser_use_deprecated_uidplus_data: false, + }) + response = parser.parse("A004 OK [COPYUID 1 101 1] Done\r\n") + assert_instance_of Net::IMAP::CopyUIDData, response.data.code.data + end + end From c67470092e53d5f8d1f8d47c80450dd7b5995302 Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 8 Jan 2025 22:06:47 -0500 Subject: [PATCH 31/34] =?UTF-8?q?=F0=9F=94=92=20Limit=20exponential=20memo?= =?UTF-8?q?ry=20usage=20to=20parse=20uid-set?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UID sets in UIDPlusData are stored as arrays of UIDs. In common scenarios, copying between one and a few hundred emails at a time, this is barely noticable. But the memory use expands _exponentially_. This should not be an issue for _trusted_ servers, and (I assume) compromised servers will be more interested in evading detection and stealing your credentials and your email than in causing client Denial of Service. Nevertheless, this is a very simple DoS attack against clients connecting to untrusted servers (for example, a service that connects to user-specified servers). For example, assuming a 64-bit architecture, considering only the data in the two arrays, assuming the arrays' internal capacity is no more than needed, and ignoring the fixed cost of the response structs: * 32 bytes expands to ~160KB (about 5000 times more): `"* OK [COPYUID 1 1:9999 1:9999]\r\n"` * 40 bytes expands to ~1.6GB (about 50 million times more): `"* OK [COPYUID 1 1:99999999 1:99999999]\r\n"` * In the worst scenario (uint32 max), 44 bytes expands to 64GiB in memory, using over 1.5 billion times more to store than to send: `"* OK [COPYUID 1 1:4294967295 1:4294967295]\r\n"` ---- The preferred fix is to store `uid-set` as a SequenceSet, not an array. Unfortunately, this is not fully backwards compatible. For v0.4 and v0.5, use `Config#parser_use_deprecated_uidplus_data` to false to use AppendUIDData and CopyUIDData instead of UIDPlusData. Unless you are _using_ UIDPLUS, this is completely safe. v0.6 will drop UIDPlusData. ---- The simplest _partial_ fix (preserving full backward compatibility) is to raise an error when the number of UIDs goes over some threshold, and continue using arrays inside UIDPlusData. For v0.3.x (and in this commit) the maximum count is hard-coded to 10,000. This is high enough that it should almost never be triggered by normal usage, and low enough to be a less extreme problem. For v0.4 and v0.5, the next commit will make the maximum array size configurable, with a much lower default: 1000 for 0.4 and 100 for 0.5. These are low enough that they are _unlikely_ to cause a problem, but 0.4 and 0.5 can also use the newer AppendUIDData and CopyUIDData classes. However, because unhandled responses are stored on the `#responses` hash, this can still be a problem. A malicious server could repeatedly use 160Kb of client memory by sending only 32 bytes in a loop. To fully solve this problem, a response handler must be added to prune excessive APPENDUID/COPYUID responses as they are received. Because unhandled responses have always been retained, managing unhandled responses is already documented as necessary for long-lived connections. --- lib/net/imap/response_parser.rb | 15 ++++++++++++--- test/net/imap/test_imap_response_parser.rb | 10 ++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index 71c08e7b..826ca527 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -8,6 +8,8 @@ class IMAP < Protocol # Parses an \IMAP server response. class ResponseParser + MAX_UID_SET_SIZE = 10_000 + include ParserUtils extend ParserUtils::Generator @@ -2023,9 +2025,16 @@ def CopyUID(...) DeprecatedUIDPlus(...) || CopyUIDData.new(...) end # TODO: remove this code in the v0.6.0 release def DeprecatedUIDPlus(validity, src_uids = nil, dst_uids) return unless config.parser_use_deprecated_uidplus_data - src_uids &&= src_uids.each_ordered_number.to_a - dst_uids = dst_uids.each_ordered_number.to_a - UIDPlusData.new(validity, src_uids, dst_uids) + compact_uid_sets = [src_uids, dst_uids].compact + count = compact_uid_sets.map { _1.count_with_duplicates }.max + max = MAX_UID_SET_SIZE + if count <= max + src_uids &&= src_uids.each_ordered_number.to_a + dst_uids = dst_uids.each_ordered_number.to_a + UIDPlusData.new(validity, src_uids, dst_uids) + else + parse_error("uid-set is too large: %d > %d", count, max) + end end ADDRESS_REGEXP = /\G diff --git a/test/net/imap/test_imap_response_parser.rb b/test/net/imap/test_imap_response_parser.rb index 7a70c02b..33427e6a 100644 --- a/test/net/imap/test_imap_response_parser.rb +++ b/test/net/imap/test_imap_response_parser.rb @@ -215,6 +215,11 @@ def test_fetch_binary_and_binary_size parser = Net::IMAP::ResponseParser.new(config: { parser_use_deprecated_uidplus_data: true, }) + assert_raise_with_message Net::IMAP::ResponseParseError, /uid-set is too large/ do + parser.parse( + "A004 OK [APPENDUID 1 10000:20000,1] Done\r\n" + ) + end response = parser.parse("A004 OK [APPENDUID 1 101:200] Done\r\n") uidplus = response.data.code.data assert_instance_of Net::IMAP::UIDPlusData, uidplus @@ -263,6 +268,11 @@ def test_fetch_binary_and_binary_size parser = Net::IMAP::ResponseParser.new(config: { parser_use_deprecated_uidplus_data: true, }) + assert_raise_with_message Net::IMAP::ResponseParseError, /uid-set is too large/ do + parser.parse( + "A004 OK [copyUID 1 10000:20000,1 1:10001] Done\r\n" + ) + end response = parser.parse("A004 OK [copyUID 1 101:200 1:100] Done\r\n") uidplus = response.data.code.data assert_instance_of Net::IMAP::UIDPlusData, uidplus From 2f58d020580176ed13fcd1e571ab7bc0e1e8f155 Mon Sep 17 00:00:00 2001 From: nick evans Date: Sun, 19 Jan 2025 12:03:19 -0500 Subject: [PATCH 32/34] =?UTF-8?q?=F0=9F=94=A7=20Add=20config=20option=20fo?= =?UTF-8?q?r=20max=20UIDPlusData=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A parser error will be raised when a `uid-set` contains more numbers than `config.parser_max_deprecated_uidplus_data_size`. --- lib/net/imap/config.rb | 34 ++++++++++++++++++++++ lib/net/imap/response_parser.rb | 4 +-- test/net/imap/test_imap_response_parser.rb | 23 +++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/lib/net/imap/config.rb b/lib/net/imap/config.rb index d5b0975a..7412fd84 100644 --- a/lib/net/imap/config.rb +++ b/lib/net/imap/config.rb @@ -291,6 +291,12 @@ def self.[](config) # CopyUIDData for +COPYUID+ response codes, and UIDPlusData or # AppendUIDData for +APPENDUID+ response codes. # + # UIDPlusData stores its data in arrays of numbers, which is vulnerable to + # a memory exhaustion denial of service attack from an untrusted or + # compromised server. Set this option to +false+ to completely block this + # vulnerability. Otherwise, parser_max_deprecated_uidplus_data_size + # mitigates this vulnerability. + # # AppendUIDData and CopyUIDData are _mostly_ backward-compatible with # UIDPlusData. Most applications should be able to upgrade with little # or no changes. @@ -313,6 +319,30 @@ def self.[](config) true, false ] + # The maximum +uid-set+ size that ResponseParser will parse into + # deprecated UIDPlusData. This limit only applies when + # parser_use_deprecated_uidplus_data is not +false+. + # + # (Parser support for +UIDPLUS+ added in +v0.3.2+.) + # + # Support for limiting UIDPlusData to a maximum size was added in + # +v0.3.8+, +v0.4.19+, and +v0.5.6+. + # + # UIDPlusData will be removed in +v0.6+. + # + # ==== Versioned Defaults + # + # Because this limit guards against a remote server causing catastrophic + # memory exhaustion, the versioned default (used by #load_defaults) also + # applies to versions without the feature. + # + # * +0.3+ and prior: 10,000 + # * +0.4+: 1,000 + # * +0.5+: 100 + # * +0.6+: 0 + # + attr_accessor :parser_max_deprecated_uidplus_data_size, type: Integer + # Creates a new config object and initialize its attribute with +attrs+. # # If +parent+ is not given, the global config is used by default. @@ -394,6 +424,7 @@ def defaults_hash enforce_logindisabled: true, responses_without_block: :warn, parser_use_deprecated_uidplus_data: true, + parser_max_deprecated_uidplus_data_size: 100, ).freeze @global = default.new @@ -406,6 +437,7 @@ def defaults_hash responses_without_block: :silence_deprecation_warning, enforce_logindisabled: false, parser_use_deprecated_uidplus_data: true, + parser_max_deprecated_uidplus_data_size: 10_000, ).freeze version_defaults[0.0] = Config[0] version_defaults[0.1] = Config[0] @@ -414,6 +446,7 @@ def defaults_hash version_defaults[0.4] = Config[0.3].dup.update( sasl_ir: true, + parser_max_deprecated_uidplus_data_size: 1000, ).freeze version_defaults[0.5] = Config[:current] @@ -421,6 +454,7 @@ def defaults_hash version_defaults[0.6] = Config[0.5].dup.update( responses_without_block: :frozen_dup, parser_use_deprecated_uidplus_data: false, + parser_max_deprecated_uidplus_data_size: 0, ).freeze version_defaults[:next] = Config[0.6] version_defaults[:future] = Config[:next] diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index 826ca527..6fb0d958 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -8,8 +8,6 @@ class IMAP < Protocol # Parses an \IMAP server response. class ResponseParser - MAX_UID_SET_SIZE = 10_000 - include ParserUtils extend ParserUtils::Generator @@ -2027,7 +2025,7 @@ def DeprecatedUIDPlus(validity, src_uids = nil, dst_uids) return unless config.parser_use_deprecated_uidplus_data compact_uid_sets = [src_uids, dst_uids].compact count = compact_uid_sets.map { _1.count_with_duplicates }.max - max = MAX_UID_SET_SIZE + max = config.parser_max_deprecated_uidplus_data_size if count <= max src_uids &&= src_uids.each_ordered_number.to_a dst_uids = dst_uids.each_ordered_number.to_a diff --git a/test/net/imap/test_imap_response_parser.rb b/test/net/imap/test_imap_response_parser.rb index 33427e6a..b1e21215 100644 --- a/test/net/imap/test_imap_response_parser.rb +++ b/test/net/imap/test_imap_response_parser.rb @@ -214,12 +214,22 @@ def test_fetch_binary_and_binary_size test "APPENDUID with parser_use_deprecated_uidplus_data = true" do parser = Net::IMAP::ResponseParser.new(config: { parser_use_deprecated_uidplus_data: true, + parser_max_deprecated_uidplus_data_size: 10_000, }) assert_raise_with_message Net::IMAP::ResponseParseError, /uid-set is too large/ do parser.parse( "A004 OK [APPENDUID 1 10000:20000,1] Done\r\n" ) end + response = parser.parse("A004 OK [APPENDUID 1 100:200] Done\r\n") + uidplus = response.data.code.data + assert_equal 101, uidplus.assigned_uids.size + parser.config.parser_max_deprecated_uidplus_data_size = 100 + assert_raise_with_message Net::IMAP::ResponseParseError, /uid-set is too large/ do + parser.parse( + "A004 OK [APPENDUID 1 100:200] Done\r\n" + ) + end response = parser.parse("A004 OK [APPENDUID 1 101:200] Done\r\n") uidplus = response.data.code.data assert_instance_of Net::IMAP::UIDPlusData, uidplus @@ -229,6 +239,7 @@ def test_fetch_binary_and_binary_size test "APPENDUID with parser_use_deprecated_uidplus_data = false" do parser = Net::IMAP::ResponseParser.new(config: { parser_use_deprecated_uidplus_data: false, + parser_max_deprecated_uidplus_data_size: 10_000_000, }) response = parser.parse("A004 OK [APPENDUID 1 10] Done\r\n") assert_instance_of Net::IMAP::AppendUIDData, response.data.code.data @@ -267,12 +278,23 @@ def test_fetch_binary_and_binary_size test "COPYUID with parser_use_deprecated_uidplus_data = true" do parser = Net::IMAP::ResponseParser.new(config: { parser_use_deprecated_uidplus_data: true, + parser_max_deprecated_uidplus_data_size: 10_000, }) assert_raise_with_message Net::IMAP::ResponseParseError, /uid-set is too large/ do parser.parse( "A004 OK [copyUID 1 10000:20000,1 1:10001] Done\r\n" ) end + response = parser.parse("A004 OK [copyUID 1 100:200 1:101] Done\r\n") + uidplus = response.data.code.data + assert_equal 101, uidplus.assigned_uids.size + assert_equal 101, uidplus.source_uids.size + parser.config.parser_max_deprecated_uidplus_data_size = 100 + assert_raise_with_message Net::IMAP::ResponseParseError, /uid-set is too large/ do + parser.parse( + "A004 OK [copyUID 1 100:200 1:101] Done\r\n" + ) + end response = parser.parse("A004 OK [copyUID 1 101:200 1:100] Done\r\n") uidplus = response.data.code.data assert_instance_of Net::IMAP::UIDPlusData, uidplus @@ -283,6 +305,7 @@ def test_fetch_binary_and_binary_size test "COPYUID with parser_use_deprecated_uidplus_data = false" do parser = Net::IMAP::ResponseParser.new(config: { parser_use_deprecated_uidplus_data: false, + parser_max_deprecated_uidplus_data_size: 10_000_000, }) response = parser.parse("A004 OK [COPYUID 1 101 1] Done\r\n") assert_instance_of Net::IMAP::CopyUIDData, response.data.code.data From e58aff64d55dda4215fa0cfd7f4d1ea7b9ca51ba Mon Sep 17 00:00:00 2001 From: nick evans Date: Wed, 22 Jan 2025 17:54:41 -0500 Subject: [PATCH 33/34] =?UTF-8?q?=F0=9F=94=A7=20Add=20`:up=5Fto=5Fmax=5Fsi?= =?UTF-8?q?ze`=20config=20for=20UIDPlusData?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When `parser_use_deprecated_uidplus_data` is set to `:up_to_max_size`, ResponseParser uses UIDPlusData when the `uid-set` size is below `parser_max_deprecated_uidplus_data_size`. Above that size, ResponseParser uses AppendUIDData or CopyUIDData. This option is now the default for v0.5. --- lib/net/imap/config.rb | 9 ++++++-- lib/net/imap/response_parser.rb | 2 +- test/net/imap/test_imap_response_parser.rb | 24 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/net/imap/config.rb b/lib/net/imap/config.rb index 7412fd84..1e0300c5 100644 --- a/lib/net/imap/config.rb +++ b/lib/net/imap/config.rb @@ -313,10 +313,15 @@ def self.[](config) # [+true+ (original default)] # ResponseParser only uses UIDPlusData. # + # [+:up_to_max_size+ (default since +v0.5.6+)] + # ResponseParser uses UIDPlusData when the +uid-set+ size is below + # parser_max_deprecated_uidplus_data_size. Above that size, + # ResponseParser uses AppendUIDData or CopyUIDData. + # # [+false+ (planned default for +v0.6+)] # ResponseParser _only_ uses AppendUIDData and CopyUIDData. attr_accessor :parser_use_deprecated_uidplus_data, type: [ - true, false + true, :up_to_max_size, false ] # The maximum +uid-set+ size that ResponseParser will parse into @@ -423,7 +428,7 @@ def defaults_hash sasl_ir: true, enforce_logindisabled: true, responses_without_block: :warn, - parser_use_deprecated_uidplus_data: true, + parser_use_deprecated_uidplus_data: :up_to_max_size, parser_max_deprecated_uidplus_data_size: 100, ).freeze diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index 6fb0d958..03e54b20 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -2030,7 +2030,7 @@ def DeprecatedUIDPlus(validity, src_uids = nil, dst_uids) src_uids &&= src_uids.each_ordered_number.to_a dst_uids = dst_uids.each_ordered_number.to_a UIDPlusData.new(validity, src_uids, dst_uids) - else + elsif config.parser_use_deprecated_uidplus_data != :up_to_max_size parse_error("uid-set is too large: %d > %d", count, max) end end diff --git a/test/net/imap/test_imap_response_parser.rb b/test/net/imap/test_imap_response_parser.rb index b1e21215..fa6ba757 100644 --- a/test/net/imap/test_imap_response_parser.rb +++ b/test/net/imap/test_imap_response_parser.rb @@ -236,6 +236,17 @@ def test_fetch_binary_and_binary_size assert_equal 100, uidplus.assigned_uids.size end + test "APPENDUID with parser_use_deprecated_uidplus_data = :up_to_max_size" do + parser = Net::IMAP::ResponseParser.new(config: { + parser_use_deprecated_uidplus_data: :up_to_max_size, + parser_max_deprecated_uidplus_data_size: 100 + }) + response = parser.parse("A004 OK [APPENDUID 1 101:200] Done\r\n") + assert_instance_of Net::IMAP::UIDPlusData, response.data.code.data + response = parser.parse("A004 OK [APPENDUID 1 100:200] Done\r\n") + assert_instance_of Net::IMAP::AppendUIDData, response.data.code.data + end + test "APPENDUID with parser_use_deprecated_uidplus_data = false" do parser = Net::IMAP::ResponseParser.new(config: { parser_use_deprecated_uidplus_data: false, @@ -302,6 +313,19 @@ def test_fetch_binary_and_binary_size assert_equal 100, uidplus.source_uids.size end + test "COPYUID with parser_use_deprecated_uidplus_data = :up_to_max_size" do + parser = Net::IMAP::ResponseParser.new(config: { + parser_use_deprecated_uidplus_data: :up_to_max_size, + parser_max_deprecated_uidplus_data_size: 100 + }) + response = parser.parse("A004 OK [COPYUID 1 101:200 1:100] Done\r\n") + copyuid = response.data.code.data + assert_instance_of Net::IMAP::UIDPlusData, copyuid + response = parser.parse("A004 OK [COPYUID 1 100:200 1:101] Done\r\n") + copyuid = response.data.code.data + assert_instance_of Net::IMAP::CopyUIDData, copyuid + end + test "COPYUID with parser_use_deprecated_uidplus_data = false" do parser = Net::IMAP::ResponseParser.new(config: { parser_use_deprecated_uidplus_data: false, From 62710b905d5672dc3dcc6d6774c1863a46e4be2b Mon Sep 17 00:00:00 2001 From: nick evans Date: Fri, 7 Feb 2025 16:26:35 -0500 Subject: [PATCH 34/34] =?UTF-8?q?=F0=9F=94=96=20Bump=20version=20to=200.5.?= =?UTF-8?q?6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 840e9c3e..1729f0b7 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -744,7 +744,7 @@ module Net # * {IMAP URLAUTH Authorization Mechanism Registry}[https://www.iana.org/assignments/urlauth-authorization-mechanism-registry/urlauth-authorization-mechanism-registry.xhtml] # class IMAP < Protocol - VERSION = "0.5.5" + VERSION = "0.5.6" # Aliases for supported capabilities, to be used with the #enable command. ENABLE_ALIASES = {