From 643f2f9eabc69b672f57eda856b5c4ae5d82a311 Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Fri, 20 Mar 2015 18:41:15 -0700 Subject: [PATCH 1/2] Add --max-issues argument to limit requests This addresses issue 71 by providing a configurable limit on the number of issues to retrieve. By default this limits the number of issues to 500 which should help users stay under the GitHub API limit and keep change logs from growing ridiculous in size for older repos. --- README.md | 1 + lib/github_changelog_generator.rb | 1 + lib/github_changelog_generator/parser.rb | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/README.md b/README.md index ccc650ae4..a97042c19 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ Type `github_changelog_generator --help` for detailed usage. --[no-]compare-link Include compare link between older version and newer version. Default is true --include-labels x,y,z Issues only with that labels will be included to changelog. Default is 'bug,enhancement' --exclude-labels x,y,z Issues with that labels will be always excluded from changelog. Default is 'duplicate,question,invalid,wontfix' + --max-issues [NUMBER] Max number of issues to fetch from GitHub. Default is 500. --github-site [URL] The Enterprise Github site on which your project is hosted. --github-api [URL] The enterprise endpoint to use for your Github API. -v, --version Print version number diff --git a/lib/github_changelog_generator.rb b/lib/github_changelog_generator.rb index 3e5171766..03fcdb295 100755 --- a/lib/github_changelog_generator.rb +++ b/lib/github_changelog_generator.rb @@ -596,6 +596,7 @@ def fetch_issues_and_pull_requests page_i += PER_PAGE_NUMBER print "Fetching issues... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r" issues.concat(page) + break if issues.length >= @options[:max_issues] end rescue puts "Warning: GitHub API rate limit exceed (5000 per hour), change log may not contain some issues.".yellow diff --git a/lib/github_changelog_generator/parser.rb b/lib/github_changelog_generator/parser.rb index b23eead4c..f86d4c31a 100644 --- a/lib/github_changelog_generator/parser.rb +++ b/lib/github_changelog_generator/parser.rb @@ -24,6 +24,7 @@ def self.parse_options :enhancement_prefix => '**Implemented enhancements:**', :author => true, :filter_issues_by_milestone => true, + :max_issues => 500, :compare_link => true, :unreleased => true, :unreleased_label => 'Unreleased', @@ -83,6 +84,9 @@ def self.parse_options opts.on('--exclude-labels x,y,z', Array, 'Issues with that labels will be always excluded from changelog. Default is \'duplicate,question,invalid,wontfix\'') do |list| options[:exclude_labels] = list end + opts.on('--max-issues [NUMBER]', Integer, 'Max number of issues to fetch from GitHub. Default is 500') do |max| + options[:max_issues] = max + end opts.on('--github-site [URL]', 'The Enterprise Github site on which your project is hosted.') do |last| options[:github_site] = last end From 856ad3a742f503cb082b46106c2982ecbe39251a Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Mon, 23 Mar 2015 08:13:44 -0700 Subject: [PATCH 2/2] Improved --max-issues docs and messaging - defaulted max issues to be unlimited in number allowing user to optionally limit issues returned - added FAQ section on rate limiting and API rate limit exceeded warning --- README.md | 11 +- lib/github_changelog_generator.rb | 136 +++++++++++------------ lib/github_changelog_generator/parser.rb | 4 +- 3 files changed, 77 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index a97042c19..12cb1d512 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Type `github_changelog_generator --help` for detailed usage. --[no-]compare-link Include compare link between older version and newer version. Default is true --include-labels x,y,z Issues only with that labels will be included to changelog. Default is 'bug,enhancement' --exclude-labels x,y,z Issues with that labels will be always excluded from changelog. Default is 'duplicate,question,invalid,wontfix' - --max-issues [NUMBER] Max number of issues to fetch from GitHub. Default is 500. + --max-issues [NUMBER] Max number of issues to fetch from GitHub. Default is unlimited. --github-site [URL] The Enterprise Github site on which your project is hosted. --github-api [URL] The enterprise endpoint to use for your Github API. -v, --version Print version number @@ -187,6 +187,15 @@ I think, that GitHub Releases is more for end-users. But `CHANGELOG.md` could stay in the repo for developers with detailed list of changes. And it's nothing bad to combine GitHub Releases and `CHANGELOG.md` file together in that manner. +- ***I received a warning: GitHub API rate limit exceed, what does this mean?*** + +GitHub [limits the number of API requests](https://developer.github.com/v3/#rate-limiting) you can make in an hour. You can make up to 5,000 requests per hour. For unauthenticated requests, the rate limit allows you to make up to 60 requests per hour. Unauthenticated requests are associated with your IP address, and not the user making requests. + +If you're seeing this warning: + +1. Make sure you're providing an OAuth token so you're not anonymously making requests. This will increase the number of requests from 60 to 5000 per hour. +2. You probably have a large repo with lots of issues/PRs. You can use the `--max-issues NUM` argument to limit the number of issues that are pulled back. For example: `--max-issues 1000` + ## Contributing 1. Create an issue to discuss about your idea diff --git a/lib/github_changelog_generator.rb b/lib/github_changelog_generator.rb index 03fcdb295..84e6cee8e 100755 --- a/lib/github_changelog_generator.rb +++ b/lib/github_changelog_generator.rb @@ -15,6 +15,8 @@ class ChangelogGenerator attr_accessor :options, :all_tags, :github PER_PAGE_NUMBER = 30 + GH_RATE_LIMIT_EXCEEDED_MSG = 'Warning: GitHub API rate limit exceed (5000 per hour), change log may not ' + + 'contain some issues. You can limit the number of issues fetched using the `--max-issues NUM` argument' def initialize @@ -27,11 +29,11 @@ def initialize github_options[:endpoint] = options[:github_endpoint] unless options[:github_endpoint].nil? github_options[:site] = options[:github_endpoint] unless options[:github_site].nil? -begin - @github = Github.new github_options -rescue - puts "Warning: GitHub API rate limit exceed (5000 per hour), change log may not contain some issues.".yellow - end + begin + @github = Github.new github_options + rescue + puts GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end @generator = Generator.new(@options) @@ -92,11 +94,11 @@ def find_closed_date_by_commit(issue) issue[:actual_date] = issue[:closed_at] else begin -begin - commit = @github.git_data.commits.get @options[:user], @options[:project], event[:commit_id] -rescue - puts "Warning: GitHub API rate limit exceed (5000 per hour), change log may not contain some issues.".yellow - end + begin + commit = @github.git_data.commits.get @options[:user], @options[:project], event[:commit_id] + rescue + puts GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end issue[:actual_date] = commit[:author][:date] rescue puts "Warning: can't fetch commit #{event[:commit_id]} probably it referenced from another repo." @@ -119,20 +121,18 @@ def fetch_merged_at_pull_requests print "Fetching merged dates...\r" end pull_requests = [] -begin - response = @github.pull_requests.list @options[:user], @options[:project], :state => 'closed' - page_i = 0 - response.each_page do |page| - page_i += PER_PAGE_NUMBER - count_pages = response.count_pages - print "Fetching merged dates... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r" - pull_requests.concat(page) - end -rescue - puts "Warning: GitHub API rate limit exceed (5000 per hour), change log may not contain some issues.".yellow - end - - + begin + response = @github.pull_requests.list @options[:user], @options[:project], :state => 'closed' + page_i = 0 + response.each_page do |page| + page_i += PER_PAGE_NUMBER + count_pages = response.count_pages + print "Fetching merged dates... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r" + pull_requests.concat(page) + end + rescue + puts GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end print " \r" @@ -290,11 +290,9 @@ def fetch_tags_dates end def is_megred(number) -begin - @github.pull_requests.merged? @options[:user], @options[:project], number -rescue - puts "Warning: GitHub API rate limit exceed (5000 per hour), change log may not contain some issues.".yellow - end + @github.pull_requests.merged? @options[:user], @options[:project], number + rescue + puts GH_RATE_LIMIT_EXCEEDED_MSG.yellow end def get_all_tags @@ -306,23 +304,21 @@ def get_all_tags tags = [] begin - response = @github.repos.tags @options[:user], @options[:project] - page_i = 0 - count_pages = response.count_pages - response.each_page do |page| - page_i += PER_PAGE_NUMBER - print "Fetching tags... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r" - tags.concat(page) - end - print " \r" - if @options[:verbose] - puts "Found #{tags.count} tags" - end -rescue - puts "Warning: GitHub API rate limit exceed (5000 per hour), change log may not contain some issues.".yellow - end - - + response = @github.repos.tags @options[:user], @options[:project] + page_i = 0 + count_pages = response.count_pages + response.each_page do |page| + page_i += PER_PAGE_NUMBER + print "Fetching tags... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r" + tags.concat(page) + end + print " \r" + if @options[:verbose] + puts "Found #{tags.count} tags" + end + rescue + puts GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end tags end @@ -536,11 +532,11 @@ def get_time_of_tag(tag_name, tag_times_hash = @tag_times_hash) return @tag_times_hash[tag_name['name']] end -begin - github_git_data_commits_get = @github.git_data.commits.get @options[:user], @options[:project], tag_name['commit']['sha'] -rescue - puts "Warning: GitHub API rate limit exceed (5000 per hour), change log may not contain some issues.".yellow - end + begin + github_git_data_commits_get = @github.git_data.commits.get @options[:user], @options[:project], tag_name['commit']['sha'] + rescue + puts GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end time_string = github_git_data_commits_get['committer']['date'] @tag_times_hash[tag_name['name']] = Time.parse(time_string) end @@ -588,21 +584,19 @@ def fetch_issues_and_pull_requests end issues = [] -begin - response = @github.issues.list user: @options[:user], repo: @options[:project], state: 'closed', filter: 'all', labels: nil - page_i = 0 - count_pages = response.count_pages - response.each_page do |page| - page_i += PER_PAGE_NUMBER - print "Fetching issues... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r" - issues.concat(page) - break if issues.length >= @options[:max_issues] - end -rescue - puts "Warning: GitHub API rate limit exceed (5000 per hour), change log may not contain some issues.".yellow - end - - + begin + response = @github.issues.list user: @options[:user], repo: @options[:project], state: 'closed', filter: 'all', labels: nil + page_i = 0 + count_pages = response.count_pages + response.each_page do |page| + page_i += PER_PAGE_NUMBER + print "Fetching issues... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r" + issues.concat(page) + break if @options[:max_issues] && issues.length >= @options[:max_issues] + end + rescue + puts GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end print " \r" @@ -644,11 +638,11 @@ def fetch_events_async(issues) issues.each_slice(max_thread_number) { |issues_slice| issues_slice.each { |issue| threads << Thread.new { -begin - obj = @github.issues.events.list user: @options[:user], repo: @options[:project], issue_number: issue['number'] -rescue - puts "Warning: GitHub API rate limit exceed (5000 per hour), change log may not contain some issues.".yellow - end + begin + obj = @github.issues.events.list user: @options[:user], repo: @options[:project], issue_number: issue['number'] + rescue + puts GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end issue[:events] = obj.body print "Fetching events for issues and PR: #{i+1}/#{@issues.count + @pull_requests.count}\r" i +=1 diff --git a/lib/github_changelog_generator/parser.rb b/lib/github_changelog_generator/parser.rb index f86d4c31a..49320e131 100644 --- a/lib/github_changelog_generator/parser.rb +++ b/lib/github_changelog_generator/parser.rb @@ -24,7 +24,7 @@ def self.parse_options :enhancement_prefix => '**Implemented enhancements:**', :author => true, :filter_issues_by_milestone => true, - :max_issues => 500, + :max_issues => nil, :compare_link => true, :unreleased => true, :unreleased_label => 'Unreleased', @@ -84,7 +84,7 @@ def self.parse_options opts.on('--exclude-labels x,y,z', Array, 'Issues with that labels will be always excluded from changelog. Default is \'duplicate,question,invalid,wontfix\'') do |list| options[:exclude_labels] = list end - opts.on('--max-issues [NUMBER]', Integer, 'Max number of issues to fetch from GitHub. Default is 500') do |max| + opts.on('--max-issues [NUMBER]', Integer, 'Max number of issues to fetch from GitHub. Default is unlimited') do |max| options[:max_issues] = max end opts.on('--github-site [URL]', 'The Enterprise Github site on which your project is hosted.') do |last|