diff --git a/.gitignore b/.gitignore index d9f725322..cd248d3be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ -bin/ +bin/* !bin/git-generate-changelog !bin/github_changelog_generator +!bin/gitlab_changelog_generator pkg/ coverage/ .bundle diff --git a/bin/gitlab_changelog_generator b/bin/gitlab_changelog_generator new file mode 100755 index 000000000..86ff22b9d --- /dev/null +++ b/bin/gitlab_changelog_generator @@ -0,0 +1,5 @@ +#! /usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../lib/gitlab_changelog_generator" +GitLabChangelogGenerator::ChangelogGenerator.new.run diff --git a/lib/github_changelog_generator/generator/generator.rb b/lib/github_changelog_generator/generator/generator.rb index 86ebdd7af..0e2c52be4 100644 --- a/lib/github_changelog_generator/generator/generator.rb +++ b/lib/github_changelog_generator/generator/generator.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "github_changelog_generator/octo_fetcher" +require "github_changelog_generator/gitlab_fetcher" require "github_changelog_generator/generator/generator_fetcher" require "github_changelog_generator/generator/generator_processor" require "github_changelog_generator/generator/generator_tags" @@ -34,7 +35,7 @@ class Generator def initialize(options = {}) @options = options @tag_times_hash = {} - @fetcher = GitHubChangelogGenerator::OctoFetcher.new(options) + @fetcher = options[:gitlab] ? GitLabChangelogGenerator::GitlabFetcher.new(options) : GitHubChangelogGenerator::OctoFetcher.new(options) @sections = [] end diff --git a/lib/github_changelog_generator/generator/generator_fetcher.rb b/lib/github_changelog_generator/generator/generator_fetcher.rb index 4e359f1d3..ebac1e53f 100644 --- a/lib/github_changelog_generator/generator/generator_fetcher.rb +++ b/lib/github_changelog_generator/generator/generator_fetcher.rb @@ -10,7 +10,8 @@ def fetch_events_for_issues_and_pr print "Fetching events for issues and PR: 0/#{@issues.count + @pull_requests.count}\r" if options[:verbose] # Async fetching events: - @fetcher.fetch_events_async(@issues + @pull_requests) + @fetcher.fetch_events_async(@issues) + @fetcher.fetch_events_async(@pull_requests) end # Async fetching of all tags dates @@ -73,9 +74,11 @@ def associate_tagged_prs(tags, prs, total) # fetch that. See # https://developer.github.com/v3/pulls/#get-a-single-pull-request vs. # https://developer.github.com/v3/pulls/#list-pull-requests - if pr["events"] && (event = pr["events"].find { |e| e["event"] == "merged" }) + # gitlab API has this + merge_commit_sha = try_merge_commit_sha_from_gitlab(pr) + if merge_commit_sha # Iterate tags.reverse (oldest to newest) to find first tag of each PR. - if (oldest_tag = tags.reverse.find { |tag| tag["shas_in_tag"].include?(event["commit_id"]) }) + if (oldest_tag = tags.reverse.find { |tag| tag["shas_in_tag"].include?(merge_commit_sha) }) pr["first_occurring_tag"] = oldest_tag["name"] found = true i += 1 @@ -95,6 +98,16 @@ def associate_tagged_prs(tags, prs, total) end end + def try_merge_commit_sha_from_gitlab(merge_request) + merge_commit_sha = nil + if merge_request.key?("merge_commit_sha") + merge_commit_sha = merge_request["merge_commit_sha"] + elsif merge_request["events"] && (event = merge_request["events"].find { |e| e["event"] == "merged" }) + merge_commit_sha = event["commit_id"] + end + merge_commit_sha + end + # Associate merged PRs by the HEAD of the release branch. If no # --release-branch was specified, then the github default branch is used. # @@ -157,7 +170,7 @@ def associate_rebase_comment_prs(tags, prs_left, total) # @param [Hash] issue def find_closed_date_by_commit(issue) unless issue["events"].nil? - # if it's PR -> then find "merged event", in case of usual issue -> fond closed date + # if it's PR -> then find "merged event", in case of usual issue -> find closed date compare_string = issue["merged_at"].nil? ? "closed" : "merged" # reverse! - to find latest closed event. (event goes in date order) issue["events"].reverse!.each do |event| diff --git a/lib/github_changelog_generator/gitlab_fetcher.rb b/lib/github_changelog_generator/gitlab_fetcher.rb new file mode 100644 index 000000000..721c94470 --- /dev/null +++ b/lib/github_changelog_generator/gitlab_fetcher.rb @@ -0,0 +1,443 @@ +# frozen_string_literal: true + +require "tmpdir" +require "retriable" +require "gitlab" + +module GitLabChangelogGenerator + # A Fetcher responsible for all requests to GitLab and all basic manipulation with related data + # (such as filtering, validating, e.t.c) + # + # Example: + # fetcher = GitLabChangelogGenerator::GitlabFetcher.new(options) + class GitlabFetcher + PER_PAGE_NUMBER = 100 + MAX_THREAD_NUMBER = 25 + MAX_FORBIDDEN_RETRIES = 100 + CHANGELOG_AUTH_TOKEN = "CHANGELOG_AUTH_TOKEN" + RATE_LIMIT_EXCEEDED_MSG = "Warning: Can't finish operation: GitLab API rate limit exceeded, changelog may be " \ + "missing some issues. You can limit the number of issues fetched using the `--max-issues NUM` argument." + NO_TOKEN_PROVIDED = "Warning: No token provided (-t option) and variable $CHANGELOG_AUTH_TOKEN was not found. " \ + "This script can make only 50 requests to GitLab API per hour without token!" + + # @param options [Hash] Options passed in + # @option options [String] :user GitLab username + # @option options [String] :project GitLab project + # @option options [String] :since Only issues updated at or after this time are returned. This is a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ. eg. Time.parse("2016-01-01 10:00:00").iso8601 + # @option options [Boolean] :http_cache Use ActiveSupport::Cache::FileStore to cache http requests + # @option options [Boolean] :cache_file If using http_cache, this is the cache file path + # @option options [Boolean] :cache_log If using http_cache, this is the cache log file path + def initialize(options = {}) + @options = options || {} + @user = @options[:user] + @project = @options[:project] + @since = @options[:since] + @http_cache = @options[:http_cache] + @cache_file = nil + @cache_log = nil + @commits = [] + @compares = {} + + Gitlab.sudo = nil + @client = Gitlab::Client.new(gitlab_options) + @project_id = find_project_id + end + + def find_project_id + project_id = nil + @client.project_search(@project).auto_paginate do |project| + project_id = project.id if project.namespace.name.eql? @user + end + project_id + end + + def gitlab_options + result = {} + access_token = fetch_auth_token + result[:private_token] = access_token if access_token + endpoint = @options[:github_endpoint] + result[:endpoint] = endpoint if endpoint + result + end + + DEFAULT_REQUEST_OPTIONS = { per_page: PER_PAGE_NUMBER } + + # Fetch all tags from repo + # + # @return [Array ] array of tags + def get_all_tags + print "Fetching tags...\r" if @options[:verbose] + + check_response { fetch_tags } + end + + # Fill input array with tags + # + # @return [Array ] array of tags in repo + def fetch_tags + tags = [] + + @client.tags(@project_id, DEFAULT_REQUEST_OPTIONS).auto_paginate do |new_tag| + tags << new_tag + end + print_empty_line + + if tags.empty? + GitHubChangelogGenerator::Helper.log.warn "Warning: Can't find any tags in repo. \ +Make sure, that you push tags to remote repo via 'git push --tags'" + else + GitHubChangelogGenerator::Helper.log.info "Found #{tags.count} tags" + end + tags.map { |resource| stringify_keys_deep(resource.to_hash) } + end + + def closed_pr_options + @closed_pr_options ||= { + filter: "all", labels: nil, state: "merged" + }.tap { |options| options[:since] = @since if @since } + end + + # This method fetch all closed issues pull requests (GitLab uses the term "merge requests") + # + # @return [Tuple] with (issues [Array ], pull-requests [Array ]) + def fetch_closed_issues_and_pr + issues = [] + print "Fetching closed issues...\r" if @options[:verbose] + options = { state: "closed", scope: :all } + @client.issues(@project_id, DEFAULT_REQUEST_OPTIONS.merge(options)).auto_paginate do |issue| + issue = stringify_keys_deep(issue.to_hash) + issue["body"] = issue["description"] + issue["html_url"] = issue["web_url"] + issue["number"] = issue["iid"] + issues.push(issue) + end + + print_empty_line + GitHubChangelogGenerator::Helper.log.info "Received issues: #{issues.count}" + + # separate arrays of issues and pull requests: + [issues.map { |issue| stringify_keys_deep(issue.to_hash) }, fetch_closed_pull_requests] + end + + # Fetch all pull requests. We need them to detect :merged_at parameter + # + # @return [Array ] all pull requests + def fetch_closed_pull_requests + pull_requests = [] + options = { state: "merged", scope: :all } + + @client.merge_requests(@project_id, options).auto_paginate do |new_pr| + new_pr = stringify_keys_deep(new_pr.to_hash) + # align with Github naming + new_pr["number"] = new_pr["iid"] + new_pr["html_url"] = new_pr["web_url"] + new_pr["merged_at"] = new_pr["updated_at"] + new_pr["pull_request"] = true + new_pr["user"] = { login: new_pr["author"]["username"], html_url: new_pr["author"]["web_url"] } + # to make it work with older gitlab version or repos that lived across versions + new_pr["merge_commit_sha"] = new_pr["merge_commit_sha"].nil? ? new_pr["sha"] : new_pr["merge_commit_sha"] + pull_requests << new_pr + end + + print_empty_line + + GitHubChangelogGenerator::Helper.log.info "Pull Request count: #{pull_requests.count}" + pull_requests.map { |pull_request| stringify_keys_deep(pull_request.to_hash) } + end + + # Fetch event for all issues and add them to 'events' + # + # @param [Array] issues + # @return [Void] + def fetch_events_async(issues) + i = 0 + threads = [] + options = {} + return if issues.empty? + + options[:target_type] = issues.first["merged_at"].nil? ? "issue" : "merge_request" + issue_events = [] + @client.project_events(@project_id, options).auto_paginate do |event| + event = stringify_keys_deep(event.to_hash) + # gitlab to github + event["event"] = event["action_name"] + issue_events << event + end + + issues.each_slice(MAX_THREAD_NUMBER) do |issues_slice| + issues_slice.each do |issue| + threads << Thread.new do + issue["events"] = [] + issue_events.each do |new_event| + if issue["id"] == new_event["target_id"] + if new_event["action_name"].eql? "closed" + issue["closed_at"] = issue["closed_at"].nil? ? new_event["created_at"] : issue["closed_at"] + end + issue["events"] << new_event + end + end + print_in_same_line("Fetching events for #{options[:target_type]}s: #{i + 1}/#{issues.count}") + i += 1 + end + end + threads.each(&:join) + threads = [] + end + + # to clear line from prev print + print_empty_line + + GitHubChangelogGenerator::Helper.log.info "Fetching events for issues and PR: #{i}" + end + + # Fetch comments for PRs and add them to "comments" + # + # @param prs [Array] PRs for which to fetch comments + # @return [Void] No return; PRs are updated in-place. + def fetch_comments_async(prs) + threads = [] + + prs.each_slice(MAX_THREAD_NUMBER) do |prs_slice| + prs_slice.each do |pr| + threads << Thread.new do + pr["comments"] = [] + @client.merge_request_notes(@project_id, pr["number"]) do |new_comment| + new_comment = stringify_keys_deep(new_comment.to_hash) + new_comment["body"] = new_comment["description"] + pr["comments"].push(new_comment) + end + pr["comments"] = pr["comments"].map { |comment| stringify_keys_deep(comment.to_hash) } + end + end + threads.each(&:join) + threads = [] + end + nil + end + + # Fetch tag time from repo + # + # @param [Hash] tag GitLab data item about a Tag + # + # @return [Time] time of specified tag + def fetch_date_of_tag(tag) + Time.parse(tag["commit"]["committed_date"]) + end + + # Fetch and cache comparison between two GitLab refs + # + # @param [String] older The older sha/tag/branch. + # @param [String] newer The newer sha/tag/branch. + # @return [Hash] GitLab api response for comparison. + def fetch_compare(older, newer) + unless @compares["#{older}...#{newer}"] + compare_data = check_response { @client.compare(@project_id, older, newer || "HEAD") } + compare_data = stringify_keys_deep(compare_data.to_hash) + compare_data["commits"].each do |commit| + commit["sha"] = commit["id"] + end + # TODO: do not know what the equivalent for gitlab is + if compare_data["compare_same_ref"] == true + raise StandardError, "Sha #{older} and sha #{newer} are not related; please file a github-changelog-generator issue and describe how to replicate this issue." + end + @compares["#{older}...#{newer}"] = stringify_keys_deep(compare_data.to_hash) + end + @compares["#{older}...#{newer}"] + end + + # Fetch commit for specified event + # + # @param [String] commit_id the SHA of a commit to fetch + # @return [Hash] + def fetch_commit(commit_id) + found = commits.find do |commit| + commit["sha"] == commit_id + end + if found + stringify_keys_deep(found.to_hash) + else + # cache miss; don't add to @commits because unsure of order. + check_response do + commit = @client.commit(@project_id, commit_id) + commit = stringify_keys_deep(commit.to_hash) + commit["sha"] = commit["id"] + commit + end + end + end + + # Fetch all commits + # + # @return [Array] Commits in a repo. + def commits + if @commits.empty? + @client.commits(@project_id).auto_paginate do |new_commit| + new_commit = stringify_keys_deep(new_commit.to_hash) + new_commit["sha"] = new_commit["id"] + @commits << new_commit + end + end + @commits + end + + # Return the oldest commit in a repo + # + # @return [Hash] Oldest commit in the gitlab git history. + def oldest_commit + commits.last + end + + # @return [String] Default branch of the repo + def default_branch + @default_branch ||= @client.project(@project_id)[:default_branch] + end + + # Fetch all SHAs occurring in or before a given tag and add them to + # "shas_in_tag" + # + # @param [Array] tags The array of tags. + # @return [void] No return; tags are updated in-place. + def fetch_tag_shas_async(tags) + i = 0 + threads = [] + print_in_same_line("Fetching SHAs for tags: #{i}/#{tags.count}\r") if @options[:verbose] + + tags.each_slice(MAX_THREAD_NUMBER) do |tags_slice| + tags_slice.each do |tag| + threads << Thread.new do + # Use oldest commit because comparing two arbitrary tags may be diverged + commits_in_tag = fetch_compare(oldest_commit["sha"], tag["name"]) + tag["shas_in_tag"] = commits_in_tag["commits"].collect { |commit| commit["sha"] } + print_in_same_line("Fetching SHAs for tags: #{i + 1}/#{tags.count}") if @options[:verbose] + i += 1 + end + end + threads.each(&:join) + threads = [] + end + + # to clear line from prev print + print_empty_line + + GitHubChangelogGenerator::Helper.log.info "Fetching SHAs for tags: #{i}" + nil + end + + private + + def stringify_keys_deep(indata) + case indata + when Array + indata.map do |value| + stringify_keys_deep(value) + end + when Hash + indata.each_with_object({}) do |(key, value), output| + output[key.to_s] = stringify_keys_deep(value) + end + else + indata + end + end + + # Exception raised to warn about moved repositories. + MovedPermanentlyError = Class.new(RuntimeError) + + def extract_request_args(args) + if args.size == 1 && args.first.is_a?(Hash) + args.delete_at(0) + elsif args.size > 1 && args.last.is_a?(Hash) + args.delete_at(args.length - 1) + else + {} + end + end + + # This is wrapper with rescue block + # + # @return [Object] returns exactly the same, what you put in the block, but wrap it with begin-rescue block + def check_response + Retriable.retriable(retry_options) do + yield + end + rescue MovedPermanentlyError => e + fail_with_message(e, "The repository has moved, update your configuration") + rescue Gitlab::Error::Forbidden => e + fail_with_message(e, "Exceeded retry limit") + rescue Gitlab::Error::Unauthorized => e + fail_with_message(e, "Error: wrong GitLab token") + end + + # Presents the exception, and the aborts with the message. + def fail_with_message(error, message) + GitHubChangelogGenerator::Helper.log.error("#{error.class}: #{error.message}") + sys_abort(message) + end + + # Exponential backoff + def retry_options + { + on: [Gitlab::Error::Forbidden], + tries: MAX_FORBIDDEN_RETRIES, + base_interval: sleep_base_interval, + multiplier: 1.0, + rand_factor: 0.0, + on_retry: retry_callback + } + end + + def sleep_base_interval + 1.0 + end + + def retry_callback + proc do |exception, try, elapsed_time, next_interval| + GitHubChangelogGenerator::Helper.log.warn("RETRY - #{exception.class}: '#{exception.message}'") + GitHubChangelogGenerator::Helper.log.warn("#{try} tries in #{elapsed_time} seconds and #{next_interval} seconds until the next try") + GitHubChangelogGenerator::Helper.log.warn RATE_LIMIT_EXCEEDED_MSG + GitHubChangelogGenerator::Helper.log.warn @client.rate_limit + end + end + + def sys_abort(msg) + abort(msg) + end + + # Print specified line on the same string + # + # @param [String] log_string + def print_in_same_line(log_string) + print log_string + "\r" + end + + # Print long line with spaces on same line to clear prev message + def print_empty_line + print_in_same_line(" ") + end + + # Returns AUTH token. First try to use variable, provided by --token option, + # otherwise try to fetch it from CHANGELOG_AUTH_TOKEN env variable. + # + # @return [String] + def fetch_auth_token + env_var = @options[:token].presence || ENV["CHANGELOG_AUTH_TOKEN"] + + GitHubChangelogGenerator::Helper.log.warn NO_TOKEN_PROVIDED unless env_var + + env_var + end + + # @return [String] "user/project" slug + def user_project + "#{@options[:user]}/#{@options[:project]}" + end + + # Returns Hash of all querystring variables in given URI. + # + # @param [String] uri eg. https://gitlab.example.com/api/v4/projects/43914960/repository/tags?page=37&foo=1 + # @return [Hash] of all GET variables. eg. { 'page' => 37, 'foo' => 1 } + def querystring_as_hash(uri) + Hash[URI.decode_www_form(URI(uri).query || "")] + end + end +end diff --git a/lib/github_changelog_generator/options.rb b/lib/github_changelog_generator/options.rb index 9be12de6a..e099ab8da 100644 --- a/lib/github_changelog_generator/options.rb +++ b/lib/github_changelog_generator/options.rb @@ -41,6 +41,7 @@ class Options < SimpleDelegator future_release github_endpoint github_site + gitlab header http_cache include_labels diff --git a/lib/github_changelog_generator/parser.rb b/lib/github_changelog_generator/parser.rb index 49db1da08..bc219c487 100755 --- a/lib/github_changelog_generator/parser.rb +++ b/lib/github_changelog_generator/parser.rb @@ -181,6 +181,9 @@ def self.setup_parser(options) opts.on("--github-api [URL]", "The enterprise endpoint to use for your GitHub API.") do |last| options[:github_endpoint] = last end + opts.on("--[no-]gitlab", "Use Gitlab API instead of Github. Default is false.") do |last| + options[:gitlab] = last + end opts.on("--simple-list", "Create a simple list from issues and pull requests. Default is false.") do |v| options[:simple_list] = v end @@ -260,7 +263,8 @@ def self.default_options removed_prefix: "**Removed:**", security_prefix: "**Security fixes:**", http_cache: true, - require: [] + require: [], + gitlab: false ) end end diff --git a/lib/github_changelog_generator/task.rb b/lib/github_changelog_generator/task.rb index 57a5a19f3..d06dc6451 100644 --- a/lib/github_changelog_generator/task.rb +++ b/lib/github_changelog_generator/task.rb @@ -17,7 +17,7 @@ class RakeTask < ::Rake::TaskLib compare_link include_labels exclude_labels bug_labels enhancement_labels between_tags exclude_tags exclude_tags_regex since_tag max_issues - github_site github_endpoint simple_list + github_site github_endpoint simple_list gitlab future_release release_branch verbose release_url base configure_sections add_sections] diff --git a/lib/gitlab_changelog_generator.rb b/lib/gitlab_changelog_generator.rb new file mode 100755 index 000000000..8197b5fe1 --- /dev/null +++ b/lib/gitlab_changelog_generator.rb @@ -0,0 +1,48 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "octokit" +require "faraday-http-cache" +require "logger" +require "active_support" +require "active_support/core_ext/object/blank" +require "json" +require "multi_json" +require "benchmark" + +require "github_changelog_generator/helper" +require "github_changelog_generator/options" +require "github_changelog_generator/parser" +require "github_changelog_generator/parser_file" +require "github_changelog_generator/generator/generator" +require "github_changelog_generator/version" +require "github_changelog_generator/reader" + +# The main module, where placed all classes (now, at least) +module GitLabChangelogGenerator + # Main class and entry point for this script. + class ChangelogGenerator + # Class, responsible for whole changelog generation cycle + # @return initialised instance of ChangelogGenerator + def initialize + @options = GitHubChangelogGenerator::Parser.parse_options + @options[:gitlab] = true + @generator = GitHubChangelogGenerator::Generator.new @options + end + + # The entry point of this script to generate changelog + # @raise (ChangelogGeneratorError) Is thrown when one of specified tags was not found in list of tags. + def run + log = @generator.compound_changelog + + if @options.write_to_file? + output_filename = @options[:output].to_s + File.open(output_filename, "wb") { |file| file.write(log) } + puts "Done!" + puts "Generated log placed in #{Dir.pwd}/#{output_filename}" + else + puts log + end + end + end +end diff --git a/spec/unit/gitlab_fetcher_spec.rb b/spec/unit/gitlab_fetcher_spec.rb new file mode 100644 index 000000000..1935206bb --- /dev/null +++ b/spec/unit/gitlab_fetcher_spec.rb @@ -0,0 +1,541 @@ +# frozen_string_literal: true + +describe GitLabChangelogGenerator::GitlabFetcher do + let(:options) do + { + user: "kreczko", + project: "changelog-testrepo", + github_endpoint: "https://gitlab.com/api/v4" + } + end + + let(:fetcher) { GitLabChangelogGenerator::GitlabFetcher.new(options) } + + describe "#check_response" do + context "when returns successfully" do + it "returns block value" do + expect(fetcher.send(:check_response) { 1 + 1 }).to eq(2) + end + end + + context "when raises GitLab::MissingCredentials" do + it "aborts" do + expect(fetcher).to receive(:sys_abort).with("Error: wrong AUTH token") + fetcher.send(:check_response) { raise(GitLab::MissingCredentials) } + end + end + + context "when raises GitLab::Forbidden" do + it "sleeps and retries and then aborts" do + retry_limit = GitLabChangelogGenerator::GitlabFetcher::MAX_FORBIDDEN_RETRIES - 1 + allow(fetcher).to receive(:sleep_base_interval).exactly(retry_limit).times.and_return(0) + + expect(fetcher).to receive(:sys_abort).with("Exceeded retry limit") + fetcher.send(:check_response) { raise(GitLab::Forbidden) } + end + end + end + + describe "#fetch_auth_token" do + token = GitLabChangelogGenerator::GitlabFetcher::CHANGELOG_AUTH_TOKEN + context "when token in ENV exist" do + before { stub_const("ENV", ENV.to_hash.merge(token => VALID_TOKEN)) } + subject { fetcher.send(:fetch_auth_token) } + it { is_expected.to eq(VALID_TOKEN) } + end + + context "when token in ENV is nil" do + before { stub_const("ENV", ENV.to_hash.merge(token => nil)) } + subject { fetcher.send(:fetch_auth_token) } + it { is_expected.to be_nil } + end + + context "when token in options and ENV is nil" do + let(:options) { { token: VALID_TOKEN } } + + before do + stub_const("ENV", ENV.to_hash.merge(token => nil)) + end + + subject { fetcher.send(:fetch_auth_token) } + it { is_expected.to eq(VALID_TOKEN) } + end + + context "when token in options and ENV specified" do + let(:options) { { token: VALID_TOKEN } } + + before do + stub_const("ENV", ENV.to_hash.merge(token => "no_matter_what")) + end + + subject { fetcher.send(:fetch_auth_token) } + it { is_expected.to eq(VALID_TOKEN) } + end + end + + describe "#get_all_tags" do + context "when fetch_tags returns tags" do + it "returns tags" do + mock_tags = ["tag"] + allow(fetcher).to receive(:fetch_tags).and_return(mock_tags) + expect(fetcher.get_all_tags).to eq(mock_tags) + end + end + end + + describe "#fetch_tags" do + context "when wrong token provided", :vcr do + let(:options) do + { + user: "skywinder", + project: "changelog_test", + token: INVALID_TOKEN, + github_endpoint: "https://gitlab.com/api/v4" + } + end + + it "should raise Unauthorized error" do + expect { fetcher.fetch_tags }.to raise_error SystemExit, "Error: wrong AUTH token" + end + end + + # TODO: swap for Gitlab API + context "when API call is valid", :vcr do + it "should return tags" do + expected_tags = [{ "name" => "v0.0.3", + "zipball_url" => + "https://api.github.com/repos/skywinder/changelog_test/zipball/v0.0.3", + "tarball_url" => + "https://api.github.com/repos/skywinder/changelog_test/tarball/v0.0.3", + "commit" => + { "sha" => "a0cba2b1a1ea9011ab07ee1ac140ba5a5eb8bd90", + "url" => + "https://api.github.com/repos/skywinder/changelog_test/commits/a0cba2b1a1ea9011ab07ee1ac140ba5a5eb8bd90" } }, + { "name" => "v0.0.2", + "zipball_url" => + "https://api.github.com/repos/skywinder/changelog_test/zipball/v0.0.2", + "tarball_url" => + "https://api.github.com/repos/skywinder/changelog_test/tarball/v0.0.2", + "commit" => + { "sha" => "9b35bb13dcd15b68e7bcbf10cde5eb937a54f710", + "url" => + "https://api.github.com/repos/skywinder/changelog_test/commits/9b35bb13dcd15b68e7bcbf10cde5eb937a54f710" } }, + { "name" => "v0.0.1", + "zipball_url" => + "https://api.github.com/repos/skywinder/changelog_test/zipball/v0.0.1", + "tarball_url" => + "https://api.github.com/repos/skywinder/changelog_test/tarball/v0.0.1", + "commit" => + { "sha" => "4c2d6d1ed58bdb24b870dcb5d9f2ceed0283d69d", + "url" => + "https://api.github.com/repos/skywinder/changelog_test/commits/4c2d6d1ed58bdb24b870dcb5d9f2ceed0283d69d" } }, + { "name" => "0.0.4", + "zipball_url" => + "https://api.github.com/repos/skywinder/changelog_test/zipball/0.0.4", + "tarball_url" => + "https://api.github.com/repos/skywinder/changelog_test/tarball/0.0.4", + "commit" => + { "sha" => "ece0c3ab7142b21064b885061c55ede00ef6ce94", + "url" => + "https://api.github.com/repos/skywinder/changelog_test/commits/ece0c3ab7142b21064b885061c55ede00ef6ce94" } }] + + expect(fetcher.fetch_tags).to eq(expected_tags) + end + + it "should return tags count" do + tags = fetcher.fetch_tags + expect(tags.size).to eq(4) + end + end + end + + describe "#fetch_closed_issues_and_pr" do + context "when API call is valid", :vcr do + it "returns issues" do + issues, pull_requests = fetcher.fetch_closed_issues_and_pr + expect(issues.size).to eq(7) + expect(pull_requests.size).to eq(14) + end + + it "returns issue with proper key/values" do + issues, _pull_requests = fetcher.fetch_closed_issues_and_pr + + expected_issue = { "url" => "https://api.github.com/repos/skywinder/changelog_test/issues/14", + "repository_url" => "https://api.github.com/repos/skywinder/changelog_test", + "labels_url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/14/labels{/name}", + "comments_url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/14/comments", + "events_url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/14/events", + "html_url" => "https://github.com/skywinder/changelog_test/issues/14", + "id" => 95_419_412, + "number" => 14, + "title" => "Issue closed from commit from PR", + "user" => + { "login" => "skywinder", + "id" => 3_356_474, + "avatar_url" => "https://avatars.githubusercontent.com/u/3356474?v=3", + "gravatar_id" => "", + "url" => "https://api.github.com/users/skywinder", + "html_url" => "https://github.com/skywinder", + "followers_url" => "https://api.github.com/users/skywinder/followers", + "following_url" => + "https://api.github.com/users/skywinder/following{/other_user}", + "gists_url" => "https://api.github.com/users/skywinder/gists{/gist_id}", + "starred_url" => + "https://api.github.com/users/skywinder/starred{/owner}{/repo}", + "subscriptions_url" => "https://api.github.com/users/skywinder/subscriptions", + "organizations_url" => "https://api.github.com/users/skywinder/orgs", + "repos_url" => "https://api.github.com/users/skywinder/repos", + "events_url" => "https://api.github.com/users/skywinder/events{/privacy}", + "received_events_url" => + "https://api.github.com/users/skywinder/received_events", + "type" => "User", + "site_admin" => false }, + "labels" => [], + "state" => "closed", + "locked" => false, + "assignee" => nil, + "assignees" => [], + "milestone" => nil, + "comments" => 0, + "created_at" => "2015-07-16T12:06:08Z", + "updated_at" => "2015-07-16T12:21:42Z", + "closed_at" => "2015-07-16T12:21:42Z", + "body" => "" } + + # Convert times to Time + expected_issue.each_pair do |k, v| + expected_issue[k] = Time.parse(v) if v =~ /^2015-/ + end + + expect(issues.first).to eq(expected_issue) + end + + it "returns pull request with proper key/values" do + _issues, pull_requests = fetcher.fetch_closed_issues_and_pr + + expected_pr = { "url" => "https://api.github.com/repos/skywinder/changelog_test/issues/21", + "repository_url" => "https://api.github.com/repos/skywinder/changelog_test", + "labels_url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/21/labels{/name}", + "comments_url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/21/comments", + "events_url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/21/events", + "html_url" => "https://github.com/skywinder/changelog_test/pull/21", + "id" => 124_925_759, + "number" => 21, + "title" => "Merged br (should appear in change log with #20)", + "user" => + { "login" => "skywinder", + "id" => 3_356_474, + "avatar_url" => "https://avatars.githubusercontent.com/u/3356474?v=3", + "gravatar_id" => "", + "url" => "https://api.github.com/users/skywinder", + "html_url" => "https://github.com/skywinder", + "followers_url" => "https://api.github.com/users/skywinder/followers", + "following_url" => + "https://api.github.com/users/skywinder/following{/other_user}", + "gists_url" => "https://api.github.com/users/skywinder/gists{/gist_id}", + "starred_url" => + "https://api.github.com/users/skywinder/starred{/owner}{/repo}", + "subscriptions_url" => "https://api.github.com/users/skywinder/subscriptions", + "organizations_url" => "https://api.github.com/users/skywinder/orgs", + "repos_url" => "https://api.github.com/users/skywinder/repos", + "events_url" => "https://api.github.com/users/skywinder/events{/privacy}", + "received_events_url" => + "https://api.github.com/users/skywinder/received_events", + "type" => "User", + "site_admin" => false }, + "labels" => [], + "state" => "closed", + "locked" => false, + "assignee" => nil, + "assignees" => [], + "milestone" => nil, + "comments" => 0, + "created_at" => "2016-01-05T09:24:08Z", + "updated_at" => "2016-01-05T09:26:53Z", + "closed_at" => "2016-01-05T09:24:27Z", + "pull_request" => + { "url" => "https://api.github.com/repos/skywinder/changelog_test/pulls/21", + "html_url" => "https://github.com/skywinder/changelog_test/pull/21", + "diff_url" => "https://github.com/skywinder/changelog_test/pull/21.diff", + "patch_url" => "https://github.com/skywinder/changelog_test/pull/21.patch" }, + "body" => + "to test https://github.com/skywinder/github-changelog-generator/pull/305\r\nshould appear in change log with #20" } + + # Convert times to Time + expected_pr.each_pair do |k, v| + expected_pr[k] = Time.parse(v) if v =~ /^2016-01/ + end + + expect(pull_requests.first).to eq(expected_pr) + end + + it "returns issues with labels" do + issues, _pull_requests = fetcher.fetch_closed_issues_and_pr + expected = [[], [], ["Bug"], [], ["enhancement"], ["some label"], []] + expect(issues.map { |i| i["labels"].map { |l| l["name"] } }).to eq(expected) + end + + it "returns pull_requests with labels" do + _issues, pull_requests = fetcher.fetch_closed_issues_and_pr + expected = [[], [], [], [], [], ["enhancement"], [], [], ["invalid"], [], [], [], [], ["invalid"]] + expect(pull_requests.map { |i| i["labels"].map { |l| l["name"] } }).to eq(expected) + end + end + end + + describe "#fetch_closed_pull_requests" do + context "when API call is valid", :vcr do + it "returns pull requests" do + pull_requests = fetcher.fetch_closed_pull_requests + expect(pull_requests.size).to eq(14) + end + + it "returns correct pull request keys" do + pull_requests = fetcher.fetch_closed_pull_requests + + pr = pull_requests.first + expect(pr.keys).to eq(%w[url id html_url diff_url patch_url issue_url number state locked title user body created_at updated_at closed_at merged_at merge_commit_sha assignee assignees milestone commits_url review_comments_url review_comment_url comments_url statuses_url head base _links]) + end + end + end + + describe "#fetch_events_async" do + context "when API call is valid", :vcr do + it "populates issues" do + issues = [{ "url" => "https://api.github.com/repos/skywinder/changelog_test/issues/14", + "repository_url" => "https://api.github.com/repos/skywinder/changelog_test", + "labels_url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/14/labels{/name}", + "comments_url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/14/comments", + "events_url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/14/events", + "html_url" => "https://github.com/skywinder/changelog_test/issues/14", + "id" => 95_419_412, + "number" => 14, + "title" => "Issue closed from commit from PR", + "user" => + { "login" => "skywinder", + "id" => 3_356_474, + "avatar_url" => "https://avatars.githubusercontent.com/u/3356474?v=3", + "gravatar_id" => "", + "url" => "https://api.github.com/users/skywinder", + "html_url" => "https://github.com/skywinder", + "followers_url" => "https://api.github.com/users/skywinder/followers", + "following_url" => + "https://api.github.com/users/skywinder/following{/other_user}", + "gists_url" => "https://api.github.com/users/skywinder/gists{/gist_id}", + "starred_url" => + "https://api.github.com/users/skywinder/starred{/owner}{/repo}", + "subscriptions_url" => + "https://api.github.com/users/skywinder/subscriptions", + "organizations_url" => "https://api.github.com/users/skywinder/orgs", + "repos_url" => "https://api.github.com/users/skywinder/repos", + "events_url" => "https://api.github.com/users/skywinder/events{/privacy}", + "received_events_url" => + "https://api.github.com/users/skywinder/received_events", + "type" => "User", + "site_admin" => false }, + "labels" => [], + "state" => "closed", + "locked" => false, + "assignee" => nil, + "assignees" => [], + "milestone" => nil, + "comments" => 0, + "created_at" => "2015-07-16T12:06:08Z", + "updated_at" => "2015-07-16T12:21:42Z", + "closed_at" => "2015-07-16T12:21:42Z", + "body" => "" }] + + # Check that they are blank to begin with + expect(issues.first["events"]).to be_nil + + fetcher.fetch_events_async(issues) + issue_events = issues.first["events"] + + expected_events = [{ "id" => 357_462_189, + "url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/events/357462189", + "actor" => + { "login" => "skywinder", + "id" => 3_356_474, + "avatar_url" => "https://avatars.githubusercontent.com/u/3356474?v=3", + "gravatar_id" => "", + "url" => "https://api.github.com/users/skywinder", + "html_url" => "https://github.com/skywinder", + "followers_url" => "https://api.github.com/users/skywinder/followers", + "following_url" => + "https://api.github.com/users/skywinder/following{/other_user}", + "gists_url" => "https://api.github.com/users/skywinder/gists{/gist_id}", + "starred_url" => + "https://api.github.com/users/skywinder/starred{/owner}{/repo}", + "subscriptions_url" => + "https://api.github.com/users/skywinder/subscriptions", + "organizations_url" => "https://api.github.com/users/skywinder/orgs", + "repos_url" => "https://api.github.com/users/skywinder/repos", + "events_url" => "https://api.github.com/users/skywinder/events{/privacy}", + "received_events_url" => + "https://api.github.com/users/skywinder/received_events", + "type" => "User", + "site_admin" => false }, + "event" => "referenced", + "commit_id" => "decfe840d1a1b86e0c28700de5362d3365a29555", + "commit_url" => + "https://api.github.com/repos/skywinder/changelog_test/commits/decfe840d1a1b86e0c28700de5362d3365a29555", + "created_at" => "2015-07-16T12:21:16Z" }, + { "id" => 357_462_542, + "url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/events/357462542", + "actor" => + { "login" => "skywinder", + "id" => 3_356_474, + "avatar_url" => "https://avatars.githubusercontent.com/u/3356474?v=3", + "gravatar_id" => "", + "url" => "https://api.github.com/users/skywinder", + "html_url" => "https://github.com/skywinder", + "followers_url" => "https://api.github.com/users/skywinder/followers", + "following_url" => + "https://api.github.com/users/skywinder/following{/other_user}", + "gists_url" => "https://api.github.com/users/skywinder/gists{/gist_id}", + "starred_url" => + "https://api.github.com/users/skywinder/starred{/owner}{/repo}", + "subscriptions_url" => + "https://api.github.com/users/skywinder/subscriptions", + "organizations_url" => "https://api.github.com/users/skywinder/orgs", + "repos_url" => "https://api.github.com/users/skywinder/repos", + "events_url" => "https://api.github.com/users/skywinder/events{/privacy}", + "received_events_url" => + "https://api.github.com/users/skywinder/received_events", + "type" => "User", + "site_admin" => false }, + "event" => "closed", + "commit_id" => "decfe840d1a1b86e0c28700de5362d3365a29555", + "commit_url" => + "https://api.github.com/repos/skywinder/changelog_test/commits/decfe840d1a1b86e0c28700de5362d3365a29555", + "created_at" => "2015-07-16T12:21:42Z" }] + + # Convert times to Time + expected_events.map! do |event| + event.each_pair do |k, v| + event[k] = Time.parse(v) if v =~ /^201[56]-/ + end + end + + expect(issue_events).to eq(expected_events) + end + end + end + + describe "#fetch_date_of_tag" do + context "when API call is valid", :vcr do + it "returns date" do + tag = { "name" => "v0.0.3", + "zipball_url" => + "https://api.github.com/repos/skywinder/changelog_test/zipball/v0.0.3", + "tarball_url" => + "https://api.github.com/repos/skywinder/changelog_test/tarball/v0.0.3", + "commit" => + { "sha" => "a0cba2b1a1ea9011ab07ee1ac140ba5a5eb8bd90", + "url" => + "https://api.github.com/repos/skywinder/changelog_test/commits/a0cba2b1a1ea9011ab07ee1ac140ba5a5eb8bd90" } } + + dt = fetcher.fetch_date_of_tag(tag) + expect(dt).to eq(Time.parse("2015-03-04 19:01:48 UTC")) + end + end + end + + describe "#querystring_as_hash" do + it "works on the blank URL" do + expect { fetcher.send(:querystring_as_hash, "") }.not_to raise_error + end + + it "where there are no querystring params" do + expect { fetcher.send(:querystring_as_hash, "http://example.com") }.not_to raise_error + end + + it "returns a String-keyed Hash of querystring params" do + expect(fetcher.send(:querystring_as_hash, "http://example.com/o.html?str=18&wis=12")).to include("wis" => "12", "str" => "18") + end + end + + describe "#fetch_commit" do + context "when API call is valid", :vcr do + it "returns commit" do + event = { "id" => 357_462_189, + "url" => + "https://api.github.com/repos/skywinder/changelog_test/issues/events/357462189", + "actor" => + { "login" => "skywinder", + "id" => 3_356_474, + "avatar_url" => "https://avatars.githubusercontent.com/u/3356474?v=3", + "gravatar_id" => "", + "url" => "https://api.github.com/users/skywinder", + "html_url" => "https://github.com/skywinder", + "followers_url" => "https://api.github.com/users/skywinder/followers", + "following_url" => + "https://api.github.com/users/skywinder/following{/other_user}", + "gists_url" => "https://api.github.com/users/skywinder/gists{/gist_id}", + "starred_url" => + "https://api.github.com/users/skywinder/starred{/owner}{/repo}", + "subscriptions_url" => "https://api.github.com/users/skywinder/subscriptions", + "organizations_url" => "https://api.github.com/users/skywinder/orgs", + "repos_url" => "https://api.github.com/users/skywinder/repos", + "events_url" => "https://api.github.com/users/skywinder/events{/privacy}", + "received_events_url" => + "https://api.github.com/users/skywinder/received_events", + "type" => "User", + "site_admin" => false }, + "event" => "referenced", + "commit_id" => "decfe840d1a1b86e0c28700de5362d3365a29555", + "commit_url" => + "https://api.github.com/repos/skywinder/changelog_test/commits/decfe840d1a1b86e0c28700de5362d3365a29555", + "created_at" => "2015-07-16T12:21:16Z" } + commit = fetcher.fetch_commit(event) + + expectations = [ + %w[sha decfe840d1a1b86e0c28700de5362d3365a29555], + ["url", + "https://api.github.com/repos/skywinder/changelog_test/commits/decfe840d1a1b86e0c28700de5362d3365a29555"], + # OLD API: "https://api.github.com/repos/skywinder/changelog_test/git/commits/decfe840d1a1b86e0c28700de5362d3365a29555"], + ["html_url", + "https://github.com/skywinder/changelog_test/commit/decfe840d1a1b86e0c28700de5362d3365a29555"], + ["author", + { "login" => "skywinder", "id" => 3_356_474, "avatar_url" => "https://avatars2.githubusercontent.com/u/3356474?v=4", "gravatar_id" => "", "url" => "https://api.github.com/users/skywinder", "html_url" => "https://github.com/skywinder", "followers_url" => "https://api.github.com/users/skywinder/followers", "following_url" => "https://api.github.com/users/skywinder/following{/other_user}", "gists_url" => "https://api.github.com/users/skywinder/gists{/gist_id}", "starred_url" => "https://api.github.com/users/skywinder/starred{/owner}{/repo}", "subscriptions_url" => "https://api.github.com/users/skywinder/subscriptions", "organizations_url" => "https://api.github.com/users/skywinder/orgs", "repos_url" => "https://api.github.com/users/skywinder/repos", "events_url" => "https://api.github.com/users/skywinder/events{/privacy}", "received_events_url" => "https://api.github.com/users/skywinder/received_events", "type" => "User", "site_admin" => false }], + ["committer", + { "login" => "skywinder", "id" => 3_356_474, "avatar_url" => "https://avatars2.githubusercontent.com/u/3356474?v=4", "gravatar_id" => "", "url" => "https://api.github.com/users/skywinder", "html_url" => "https://github.com/skywinder", "followers_url" => "https://api.github.com/users/skywinder/followers", "following_url" => "https://api.github.com/users/skywinder/following{/other_user}", "gists_url" => "https://api.github.com/users/skywinder/gists{/gist_id}", "starred_url" => "https://api.github.com/users/skywinder/starred{/owner}{/repo}", "subscriptions_url" => "https://api.github.com/users/skywinder/subscriptions", "organizations_url" => "https://api.github.com/users/skywinder/orgs", "repos_url" => "https://api.github.com/users/skywinder/repos", "events_url" => "https://api.github.com/users/skywinder/events{/privacy}", "received_events_url" => "https://api.github.com/users/skywinder/received_events", "type" => "User", "site_admin" => false }], + ["parents", + [{ "sha" => "7ec095e5e3caceacedabf44d0b9b10da17c92e51", + "url" => + "https://api.github.com/repos/skywinder/changelog_test/commits/7ec095e5e3caceacedabf44d0b9b10da17c92e51", + # OLD API: "https://api.github.com/repos/skywinder/changelog_test/git/commits/7ec095e5e3caceacedabf44d0b9b10da17c92e51", + "html_url" => + "https://github.com/skywinder/changelog_test/commit/7ec095e5e3caceacedabf44d0b9b10da17c92e51" }]] + ] + + expectations.each do |property, val| + expect(commit[property]).to eq(val) + end + end + end + end + + describe "#commits" do + context "when API is valid", :vcr do + subject do + fetcher.commits + end + + it "returns commits" do + expect(subject.last["sha"]).to eq("4c2d6d1ed58bdb24b870dcb5d9f2ceed0283d69d") + end + end + end +end