From 0c5291cdd6b30f382ad2cd00508fc6893cefc311 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 3 Jun 2021 11:45:59 +1200 Subject: [PATCH 1/5] Directly handle rate limiting using `client.rate_limit.resets_in`. --- .../octo_fetcher.rb | 41 ++++++------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/lib/github_changelog_generator/octo_fetcher.rb b/lib/github_changelog_generator/octo_fetcher.rb index 73e78251..e0caa638 100644 --- a/lib/github_changelog_generator/octo_fetcher.rb +++ b/lib/github_changelog_generator/octo_fetcher.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "tmpdir" -require "retriable" require "set" require "async" require "async/barrier" @@ -456,10 +455,21 @@ def iterate_pages(client, method, *arguments, parent: nil, **options) # # @return [Object] returns exactly the same, what you put in the block, but wrap it with begin-rescue block # @param [Proc] block - def check_github_response(&block) - Retriable.retriable(retry_options, &block) + def check_github_response + yield rescue MovedPermanentlyError => e fail_with_message(e, "The repository has moved, update your configuration") + rescue Octokit::TooManyRequests => e + resets_in = client.rate_limit.resets_in + Helper.log.error("#{e.class} #{e.message}; sleeping for #{resets_in}s...") + + if task = Async::Task.current? + task.sleep(resets_in) + else + sleep(resets_in) + end + + retry rescue Octokit::Forbidden => e fail_with_message(e, "Exceeded retry limit") rescue Octokit::Unauthorized => e @@ -474,31 +484,6 @@ def fail_with_message(error, message) sys_abort(message) end - # Exponential backoff - def retry_options - { - on: [Octokit::Forbidden], - tries: MAX_FORBIDDEN_RETRIES, - base_interval: sleep_base_interval, - multiplier: 1.0, - rand_factor: 0.0, - on_retry: retry_callback - } - end - - def sleep_base_interval - 1.0 - end - - def retry_callback - proc do |exception, try, elapsed_time, next_interval| - Helper.log.warn("RETRY - #{exception.class}: '#{exception.message}'") - Helper.log.warn("#{try} tries in #{elapsed_time} seconds and #{next_interval} seconds until the next try") - Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG - Helper.log.warn(client.rate_limit) - end - end - # @param [Object] msg def sys_abort(msg) abort(msg) From ef010930fb0867cab4107b39641f15ba5915f892 Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Thu, 3 Jun 2021 08:50:44 +0200 Subject: [PATCH 2/5] Linting --- lib/github_changelog_generator/octo_fetcher.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/github_changelog_generator/octo_fetcher.rb b/lib/github_changelog_generator/octo_fetcher.rb index e0caa638..4a1b25c9 100644 --- a/lib/github_changelog_generator/octo_fetcher.rb +++ b/lib/github_changelog_generator/octo_fetcher.rb @@ -463,7 +463,7 @@ def check_github_response resets_in = client.rate_limit.resets_in Helper.log.error("#{e.class} #{e.message}; sleeping for #{resets_in}s...") - if task = Async::Task.current? + if (task = Async::Task.current?) task.sleep(resets_in) else sleep(resets_in) From b97234a946cb61a0f4377baab3ee354269f11607 Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Thu, 3 Jun 2021 08:55:17 +0200 Subject: [PATCH 3/5] Spec: Drop an allow for a no-longer-implemented method --- spec/unit/octo_fetcher_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/unit/octo_fetcher_spec.rb b/spec/unit/octo_fetcher_spec.rb index 382f7be8..99e50857 100644 --- a/spec/unit/octo_fetcher_spec.rb +++ b/spec/unit/octo_fetcher_spec.rb @@ -30,7 +30,6 @@ context "when raises Octokit::Forbidden" do it "sleeps and retries and then aborts" do retry_limit = GitHubChangelogGenerator::OctoFetcher::MAX_FORBIDDEN_RETRIES - 1 - allow(fetcher).to receive(:sleep_base_interval).exactly(retry_limit).times.and_return(0) expect(fetcher).to receive(:sys_abort).with("Exceeded retry limit") fetcher.send(:check_github_response) { raise(Octokit::Forbidden) } From b78bb8c51da5b2949819fc9442f7bed9e90bd0ee Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Thu, 3 Jun 2021 09:00:44 +0200 Subject: [PATCH 4/5] Linting --- spec/unit/octo_fetcher_spec.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/unit/octo_fetcher_spec.rb b/spec/unit/octo_fetcher_spec.rb index 99e50857..0f981815 100644 --- a/spec/unit/octo_fetcher_spec.rb +++ b/spec/unit/octo_fetcher_spec.rb @@ -29,8 +29,6 @@ context "when raises Octokit::Forbidden" do it "sleeps and retries and then aborts" do - retry_limit = GitHubChangelogGenerator::OctoFetcher::MAX_FORBIDDEN_RETRIES - 1 - expect(fetcher).to receive(:sys_abort).with("Exceeded retry limit") fetcher.send(:check_github_response) { raise(Octokit::Forbidden) } end From efd586d0674078c3dd3282481376e52f5ee4b3b6 Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Thu, 3 Jun 2021 09:03:26 +0200 Subject: [PATCH 5/5] Remove unused runtime dependency retriable --- Gemfile.lock | 2 -- github_changelog_generator.gemspec | 1 - 2 files changed, 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c2a7a631..e1f4112d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,6 @@ PATH octokit (~> 4.6) rainbow (>= 2.2.1) rake (>= 10.0) - retriable (~> 3.0) GEM remote: https://rubygems.org/ @@ -95,7 +94,6 @@ GEM rainbow (3.0.0) rake (13.0.1) regexp_parser (1.8.1) - retriable (3.1.2) rexml (3.2.5) rspec (3.9.0) rspec-core (~> 3.9.0) diff --git a/github_changelog_generator.gemspec b/github_changelog_generator.gemspec index 32f3c07a..60b1da47 100644 --- a/github_changelog_generator.gemspec +++ b/github_changelog_generator.gemspec @@ -31,5 +31,4 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency("octokit", ["~> 4.6"]) spec.add_runtime_dependency "rainbow", ">= 2.2.1" spec.add_runtime_dependency "rake", ">= 10.0" - spec.add_runtime_dependency("retriable", ["~> 3.0"]) end