bin/git-sync-check.rb

#!/usr/bin/env ruby
# This is executed by `/etc/systemd/system/git-sync-check.service` as User=git
# which is triggered every 10 minutes by `/etc/systemd/system/git-sync-check.timer`.

require 'json'
require 'net/http'
require 'uri'

module Git
  # cgit bare repository
  GIT_DIR = '/var/git/ruby.git'

  # This is retried because ls-remote of GitHub sometimes fails
  Error = Class.new(StandardError)

  class << self
    def show_ref
      git('show-ref')
    end

    def ls_remote(remote)
      git('ls-remote', remote)
    end

    private

    def git(*cmd)
      out = IO.popen({ 'GIT_DIR' => GIT_DIR }, ['git', *cmd], &:read)
      unless $?.success?
        raise Git::Error.new("Failed to execute: git #{cmd.join(' ')}")
      end
      out
    end
  end
end

module Slack
  WEBHOOK_URL = File.read(File.expand_path('~git/config/slack-webhook-alerts')).chomp
  NOTIFY_CHANNELS = [
    "C5FCXFXDZ", # alerts
    "CR2QGFCAE", # alerts-emoji
  ]

  class << self
    def notify(message)
      attachment = {
        title: 'bin/git-sync-check.rb',
        title_link: 'https://github.com/ruby/git.ruby-lang.org/blob/master/bin/git-sync-check.rb',
        text: message,
        color: 'danger',
      }

      payload = { username: 'ruby/git.ruby-lang.org', attachments: [attachment] }
      NOTIFY_CHANNELS.each do |channel|
        resp = post(WEBHOOK_URL, payload: payload.merge(channel: channel))
        puts "#{resp.code} (#{resp.body}) -- #{payload.to_json} (channel: #{channel})"
      end
    end

    private

    def post(url, payload:)
      uri = URI.parse(url)
      http = Net::HTTP.new(uri.host, uri.port)
      http.use_ssl = (uri.scheme == 'https')
      http.start do
        req = Net::HTTP::Post.new(uri.path)
        req.set_form_data({ payload: payload.to_json })
        http.request(req)
      end
    end
  end
end

module GitSyncCheck
  class Errors < StandardError
    attr_reader :errors

    def initialize(errors)
      @errors = errors
      super('git-sync-check failed')
    end
  end

  def self.check_consistency
    # Quickly finish collecting facts to avoid a race condition as much as possible.
    ls_remote = Git.ls_remote('github')
    show_ref  = Git.show_ref

    # Start digesting the data after the collection.
    remote_refs = Hash[ls_remote.lines.map { |l| rev, ref = l.chomp.split("\t"); [ref, rev] }]
    local_refs  = Hash[show_ref.lines.map  { |l| rev, ref = l.chomp.split(' ');  [ref, rev] }]

    # Remove refs which are not to be checked here.
    refs = remote_refs.keys | local_refs.keys
    refs.delete('HEAD') # show-ref does not show it
    refs.delete('refs/notes/commits') # it seems too complicated to recover its inconsistency
    remote_refs.keys.each { |ref| refs.delete(ref) if ref.match(%r[\Arefs/pull/\d+/\w+\z]) } # pull requests

    # Check consistency
    errors = {}
    refs.each do |ref|
      remote_rev = remote_refs[ref]
      local_rev  = local_refs[ref]

      if remote_rev != local_rev
        errors[ref] = [remote_rev, local_rev]
      end
    end

    unless errors.empty?
      raise Errors.new(errors)
    end
  end
end

attempts = 3
begin
  GitSyncCheck.check_consistency
  puts 'SUCCUESS: Everything is consistent.'
rescue GitSyncCheck::Errors => e
  attempts -= 1
  if attempts > 0
    # Automatically fix inconsistency if it's master, but never sync random new branches.
    ref = 'refs/heads/master'
    if e.errors.key?(ref)
      remote_rev, local_rev = e.errors['refs/heads/master']
      puts "Fixing inconsistency ref:#{ref.inspect} remote:#{remote_rev.inspect} local:#{local_rev.inspect}"
      unless system('/home/git/git.ruby-lang.org/bin/update-ruby.sh', File.basename(ref))
        raise "Failed to execute update-ruby.sh for #{ref}"
      end
    end
    sleep 5
    retry
  end

  message = "FAILURE: Following inconsistencies are found.\n"
  e.errors.each do |ref, (remote_rev, local_rev)|
    message << "ref:#{ref.inspect} remote:#{remote_rev.inspect} local:#{local_rev.inspect}\n"
  end
  # Slack.notify(message)
  puts message
rescue Git::Error => e
  attempts -= 1
  if attempts > 0
    puts "Retrying #{e.class}: #{e.message} (remaining attempts: #{attempts})"
    sleep 5
    retry
  end
  Slack.notify("#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}")
rescue => e
  Slack.notify("#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}")
end