diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..93cc45f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,22 @@ +# Contributing + +## How docs.ruby-lang.org/en Works + +### Overview + +The English Ruby documentation is automatically generated and updated through a pipeline involving GitHub Actions, AWS S3, and systemd services. Here's how it all comes together: + +1. **Documentation Generation and Upload** + + - A scheduled [GitHub Action](https://github.com/ruby/actions/blob/master/.github/workflows/docs.yml) generates the documentation for all [actively maintained Ruby versions](https://www.ruby-lang.org/en/downloads/branches/) plus the master branch. Each documentation set is generated using the version of RDoc bundled with that specific Ruby version. + - *Example:* For [Ruby 3.3](https://docs.ruby-lang.org/en/3.3/), the documentation is generated with RDoc 6.6.3.1. + - The generated docs are uploaded as `ruby-docs-en-.tar.xz` to the `ftp.r-l.o/pub/ruby/doc/` S3 bucket. + - You can find these uploads at [cache.ruby-lang.org/pub/ruby/doc/](https://cache.ruby.org/pub/ruby/doc/), which is powered by [cache.r-l.o](https://github.com/ruby/cache.r-l.o). + +2. **Website Update** + + - On the server running this application, the `update-docs-en.timer` systemd timer triggers the `rdoc-static-all` service at scheduled intervals. + - Timer definition: [update-docs-en.timer](https://github.com/ruby/docs.ruby-lang.org/blob/master/provision/systemd/update-docs-en.timer) + - Service definition: [rdoc-static-all.service](https://github.com/ruby/docs.ruby-lang.org/blob/master/provision/systemd/rdoc-static-all.service) + - The `rdoc-static-all` service runs the [rdoc-static-all](https://github.com/ruby/docs.ruby-lang.org/blob/master/system/rdoc-static-all) script. + - This script downloads the latest documentation from S3 and updates the website accordingly. diff --git a/Gemfile b/Gemfile index 12d280a..af379c3 100644 --- a/Gemfile +++ b/Gemfile @@ -7,3 +7,6 @@ gem 'capistrano-bundler' # ed25519 gem 'ed25519' gem 'bcrypt_pbkdf' + +# For previewing public directory locally with `bin/server` +gem 'webrick' diff --git a/Gemfile.lock b/Gemfile.lock index 0043163..839a18c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,28 +1,42 @@ GEM remote: https://rubygems.org/ specs: - airbrussh (1.4.0) + airbrussh (1.5.3) sshkit (>= 1.6.1, != 1.7.0) - bcrypt_pbkdf (1.0.1) - capistrano (3.14.1) + base64 (0.2.0) + bcrypt_pbkdf (1.1.1) + capistrano (3.19.2) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) sshkit (>= 1.9.0) - capistrano-bundler (2.0.1) + capistrano-bundler (2.1.1) capistrano (~> 3.1) - concurrent-ruby (1.1.6) - ed25519 (1.2.4) - i18n (1.8.5) + concurrent-ruby (1.3.4) + date (3.4.1) + ed25519 (1.3.0) + i18n (1.14.6) concurrent-ruby (~> 1.0) - net-scp (3.0.0) - net-ssh (>= 2.6.5, < 7.0.0) - net-ssh (6.1.0) - rake (13.0.1) - rdoc (6.2.1) - sshkit (1.21.0) + net-scp (4.0.0) + net-ssh (>= 2.6.5, < 8.0.0) + net-sftp (4.0.0) + net-ssh (>= 5.0.0, < 8.0.0) + net-ssh (7.3.0) + ostruct (0.6.1) + psych (5.2.3) + date + stringio + rake (13.2.1) + rdoc (6.13.1) + psych (>= 4.0.0) + sshkit (1.23.2) + base64 net-scp (>= 1.1.2) + net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) + ostruct + stringio (3.1.6) + webrick (1.9.1) PLATFORMS ruby @@ -33,6 +47,7 @@ DEPENDENCIES capistrano-bundler ed25519 rdoc (>= 6.0.0.beta2) + webrick BUNDLED WITH - 2.1.2 + 2.5.17 diff --git a/README.md b/README.md index 109f8f2..fe82d1f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # docs.ruby-lang.org +## WARNING + +* This repository is inconsistent with production. +* Use only `bundle exec cap production deploy`. Do not use `ansible-playbook`. + ## Platform Environment * We use `capistrano` for deployments. @@ -7,12 +12,6 @@ * All applications run as `rurema` user. * Periodic tasks run in systemd timer (`systemctl list-timers`) -### Requisites - -* Debian 10 (buster) - * `apt install nginx groonga git bundler certbot` -* passenger nginx module: https://www.phusionpassenger.com/docs/advanced_guides/install_and_upgrade/standalone/install/oss/buster.html - ## Files and Directories * rurema-search: `/var/rubydoc` @@ -22,57 +21,37 @@ ## Files need to backup * old statically generated contents (old versions need to copy from old server): `/var/www/docs.ruby-lang.org/shared/public/{en,ja}/*` + * Versioned contents of `ja` are from since 2024-06-06. * Fastly API Key: `/home/rurema/.docs-fastly` * Slack webhook URL: `/etc/systemd/system/notify-to-slack.env` +* Mackerel API Key and plugin configurations: `/etc/mackerel-agent/mackerel-agent.conf` ## Related repos * https://github.com/ruby/rurema-search +* https://github.com/rurema/generated-documents -## Vagrant Environment - -Usage: +## Previewing `public` directory locally -``` -git clone https://github.com/ruby/docs.ruby-lang.org -git clone https://github.com/ruby/rurema-search -cd docs.ruby-lang.org -vagrant up -vagrant ssh-config >> ~/.ssh/config -bundle install -cap vagrant deploy -cp config/deploy/vagrant.rb ../rurema-search/config/deploy/vagrant.rb -cd ../rurema-search -cap vagrant deploy -cd ../docs.ruby-lang.org -vagrant ssh -sudo systemctl start rdoc-static-all.service bc-setup-all.service & -sudo systemctl status rdoc-static-all.service bc-setup-all.service bc-static-all.service update-rurema-index.service -``` - -- Open `https://localhost:10443/` in browser (ignore certificate error (`NET::ERR_CERT_AUTHORITY_INVALID`) because of using self signed certificate generated by `provision/selfsigned.yml`) -- Run `sudo systemctl start rdoc-static-all.service` to update English documents. -- Run `sudo systemctl start bc-setup-all.service` to update Japanese documents. -- Run `sudo systemctl status rdoc-static-all.service bc-setup-all.service bc-static-all.service update-rurema-index.service` to see progress. - - `Active: activating (start) since ...` means running. - - `Active: inactive (dead) since ...` means finished. +1. Install dependencies: `bundle install` +2. Run: + ``` + bin/server + ``` +3. Open http://localhost:8000/ ## Production Environment -## Capstrano +### Capstrano ``` cap production deploy ``` -### ansible +### /etc/nginx/sites-available/docs.ruby-lang.org -``` -ansible-playbook -i docs-2020, provision/playbook.yml -ansible-playbook -i docs-2020, provision/users.yml -ansible-playbook -i docs-2020, provision/rurema-search.yml -ansible-playbook -i docs-2020, provision/letsencrypt.yml -``` +Manual sync with `conf/docs.ruby-lang.org`. +Because ansible playbooks are outdated. ### /etc/mackerel-agent/mackerel-agent.conf @@ -91,5 +70,5 @@ command = ["check-file-age", "-i", "-w", "90000", "-c", "172800", "-f", "/run/do [plugin.checks.fileage-rdoc-static-all] command = ["check-file-age", "-i", "-w", "90000", "-c", "172800", "-f", "/run/docs.ruby-lang.org/rdoc-static-all.updated"] [plugin.checks.fileage-update-rurema-index] -command = ["check-file-age", "-i", "-w", "90000", "-c", "172800", "-f", "/run/docs.ruby-lang.org/update-rurema-index.updated"] +command = ["check-file-age", "-i", "-w", "90060", "-c", "172800", "-f", "/run/docs.ruby-lang.org/update-rurema-index.updated"] ``` diff --git a/Rakefile b/Rakefile deleted file mode 100644 index 4211ff5..0000000 --- a/Rakefile +++ /dev/null @@ -1,60 +0,0 @@ -require 'rdoc/task' - -versions = { - "master" => "master", - "2.7.0" => "ruby_2_7", - "2.6.0" => "ruby_2_6", - "2.5.0" => "ruby_2_5", -} - -versions.each do |version, branch_name| - source_dir = "sources/#{version}" - - directory source_dir do - sh "git clone --depth=1 --branch=#{branch_name} git://github.com/ruby/ruby.git #{source_dir}" - end - - desc "Checks out source for #{version}" - task "source:#{version}" => source_dir - - desc "Updates source for #{version}" - task "update:#{version}" => source_dir do - Dir.chdir source_dir do - sh "git fetch origin" - sh "git reset origin/#{branch_name} --hard" - end - end - - desc "Compile source for #{version}" - task "compile:#{version}" => source_dir do - Dir.chdir source_dir do - sh "make clean" if File.exists?("Makefile") - sh "autoconf && ./configure && make" - end - end - - namespace :rdoc do - lang_version = File.join("en", version) - RDoc::Task.new(lang_version) do |rdoc| - rdoc.title = "Documentation for Ruby #{version}" - rdoc.main = "README.md" - rdoc.rdoc_dir = version - rdoc.rdoc_files << source_dir - rdoc.options << "-U" - rdoc.options << "--all" - rdoc.options << "--root=#{source_dir}" - rdoc.options << "--page-dir=#{source_dir}/doc" - rdoc.options << "--encoding=UTF-8" - end - end - task "rdoc:#{version}" => ["update:#{version}", "compile:#{version}"] -end - -desc "Checks out sources for all versions" -task "source" => versions.keys.map { |version| "source:#{version}" } - -desc "Updates sources for all versions" -task "update" => versions.keys.map { |version| "update:#{version}" } - -desc "Build RDoc HTML files for all versions" -task "rdoc" => versions.keys.map { |version| "rdoc:#{version}" } diff --git a/bin/server b/bin/server new file mode 100755 index 0000000..a14c031 --- /dev/null +++ b/bin/server @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require "webrick" + +server = WEBrick::HTTPServer.new(Port: 8000, DocumentRoot: 'public') +server.start diff --git a/conf/crontab b/conf/crontab deleted file mode 100644 index 22a7051..0000000 --- a/conf/crontab +++ /dev/null @@ -1,4 +0,0 @@ -20 4 * * * cd /var/www/docs.ruby-lang.org/current; ruby system/rdoc-static-all -15 9 * * * cd /var/www/docs.ruby-lang.org/current; ruby system/bc-setup-all; ruby system/bc-static-all -15 11 * * * cd /var/www/docs.ruby-lang.org/current; system/update-rurema-index - diff --git a/conf/docs.ruby-lang.org b/conf/docs.ruby-lang.org index d13b2b3..59cfd65 100644 --- a/conf/docs.ruby-lang.org +++ b/conf/docs.ruby-lang.org @@ -1,3 +1,9 @@ + # define limit zone + #limit_req_zone $binary_remote_addr zone=search_limit:10m rate=1r/s; + limit_req_zone global zone=search_limit:10m rate=2r/s; + + passenger_max_pool_size 2; + server { listen 80; server_name doc.ruby-lang.org; @@ -37,7 +43,6 @@ server_name docs.ruby-lang.org docs-origin.ruby-lang.org; - ssl on; ssl_certificate /etc/letsencrypt/live/docs-origin.ruby-lang.org/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/docs-origin.ruby-lang.org/privkey.pem; ssl_session_timeout 1d; @@ -64,6 +69,12 @@ proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-Port 443; + # generated by + # curl -fsSL https://api.fastly.com/public-ip-list | jq -r '.addresses+.ipv6_addresses|map("set_real_ip_from "+.+";")|join("\n")' > set_real_ip_from.conf + include /etc/nginx/set_real_ip_from.conf; + real_ip_header Fastly-Client-IP; + real_ip_recursive on; + location / { root /var/www/docs.ruby-lang.org/current/public; index index.html; @@ -75,8 +86,18 @@ return 301 https://docs.ruby-lang.org/en/master$1; } + # https://github.com/ruby/docs.ruby-lang.org/issues/130 + location ~ ^/en/([^/]+)/doc/(.*) { + return 301 https://docs.ruby-lang.org/en/$1/$2; + } + + # 2.8.0 was renamed to 3.0.0 location ~ ^/(en|ja)/2\.8\.0(.*) { - return 302 https://docs.ruby-lang.org/$1/master$2; + return 301 https://docs.ruby-lang.org/$1/3.0$2; + } + + location ~ ^/(en|ja)/3\.0\.0(.*) { + return 301 https://docs.ruby-lang.org/$1/3.0$2; } location ~ ^/(en|ja)/(.+?)/ { @@ -85,7 +106,16 @@ } } + location /capi/en/master/ { + rewrite ^(.*)/$ $1/index.html; + proxy_pass https://rubyci.s3.amazonaws.com/doxygen-latest-html/; + add_header Cache-Control "public, max-age=43200, s-maxage=172800, stale-while-revalidate=86400, stale-if-error=604800"; + add_header Surrogate-Key "doxygen-latest-html"; + } + location /ja/search { + limit_req zone=search_limit burst=1 nodelay; + limit_req_status 429; proxy_pass http://localhost:9292; } @@ -107,6 +137,7 @@ server_name localhost; root /var/rubydoc/rurema-search/current/public; passenger_enabled on; + passenger_max_requests 50000; # add_header Cache-Control "private"; add_header Cache-Control "public, max-age=3600, s-maxage=172800, stale-while-revalidate=86400, stale-if-error=60"; @@ -119,7 +150,6 @@ server_name doc.ruby-lang.org; - ssl on; ssl_certificate /etc/letsencrypt/live/doc.ruby-lang.org/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/doc.ruby-lang.org/privkey.pem; ssl_session_timeout 1d; diff --git a/config/deploy.rb b/config/deploy.rb index bba6aac..f945731 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -4,7 +4,12 @@ set :default_env, { 'PATH' => '/snap/bin:$PATH', + 'DEBIAN_DISABLE_RUBYGEMS_INTEGRATION' => 'true', } -# TODO: Add 3.0.0 after 2.8.0 is changed to 3.0.0 -versions = %w[1.8.7 1.9.3 2.0.0 2.1.0 2.2.0 2.3.0 2.4.0 2.5.0 2.6.0 2.7.0 master] -set :linked_dirs, ["sources", "public/ja/latest"] + versions.map{|v| ["public/en/#{v}", "public/ja/#{v}"]}.flatten +versions = %w[ + 1.8.7 1.9.3 + 2.0.0 2.1.0 2.2.0 2.3.0 2.4.0 2.5.0 2.6.0 2.7.0 + 3.0 3.1 3.2 3.3 3.4 + master +] +set :linked_dirs, ["sources", "public/ja/3.5", "public/ja/latest"] + versions.map{|v| ["public/en/#{v}", "public/ja/#{v}"]}.flatten diff --git a/provision/playbook.yml b/provision/playbook.yml index e560f60..e661a8b 100644 --- a/provision/playbook.yml +++ b/provision/playbook.yml @@ -5,6 +5,9 @@ vars: ansible_python_interpreter: '/usr/bin/python' tasks: + - name: 'Setup groonga-apt-source' # https://groonga.org/ja/docs/install/debian.html + apt: + deb: 'https://packages.groonga.org/debian/groonga-apt-source-latest-buster.deb' - name: 'Setup packages' apt: name: @@ -85,6 +88,7 @@ src: 'systemd/{{ item }}' with_items: - 'prepend-snap-path' + - 'env-clear-before-replace.rb' - 'update-docs-en.timer' - 'rdoc-static-all.service' - 'update-docs-ja.timer' diff --git a/provision/rurema-search.yml b/provision/rurema-search.yml index 508905c..99a949f 100644 --- a/provision/rurema-search.yml +++ b/provision/rurema-search.yml @@ -32,7 +32,7 @@ dest: '/var/rubydoc/rurema-search/shared/document.yaml' content: | base_url: - http://docs.ruby-lang.org/ja/ + https://docs.ruby-lang.org/ja/ remove_dot_from_version: false tracking_id: diff --git a/provision/systemd/env-clear-before-replace.rb b/provision/systemd/env-clear-before-replace.rb new file mode 100644 index 0000000..ce8bd1f --- /dev/null +++ b/provision/systemd/env-clear-before-replace.rb @@ -0,0 +1,7 @@ +class << ENV + alias orig_replace replace + def replace(h) + clear + orig_replace(h) + end +end diff --git a/provision/systemd/prepend-snap-path b/provision/systemd/prepend-snap-path index ae2443d..1f60b76 100644 --- a/provision/systemd/prepend-snap-path +++ b/provision/systemd/prepend-snap-path @@ -1 +1,3 @@ PATH=/snap/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +DEBIAN_DISABLE_RUBYGEMS_INTEGRATION=true +RUBYOPT=-r/etc/systemd/system/env-clear-before-replace.rb diff --git a/provision/systemd/update-docs-ja.timer b/provision/systemd/update-docs-ja.timer index 078e31a..29bf518 100644 --- a/provision/systemd/update-docs-ja.timer +++ b/provision/systemd/update-docs-ja.timer @@ -2,7 +2,7 @@ Description=Update docs.ruby-lang.org/ja/ [Timer] -OnCalendar=*-*-* 0:15 +OnCalendar=*-*-* 20:15 Persistent=true Unit=bc-setup-all.service diff --git a/public/en/index.html b/public/en/index.html index 283fde8..02f2119 100644 --- a/public/en/index.html +++ b/public/en/index.html @@ -17,18 +17,25 @@ diff --git a/public/favicon.ico b/public/favicon.ico new file mode 100644 index 0000000..4a8e9e9 Binary files /dev/null and b/public/favicon.ico differ diff --git a/public/index.html b/public/index.html index 1a670b1..8b41d99 100644 --- a/public/index.html +++ b/public/index.html @@ -21,6 +21,7 @@

docs.ruby-lang.org

diff --git a/public/ja/index.html b/public/ja/index.html index 3011cb3..9a11979 100644 --- a/public/ja/index.html +++ b/public/ja/index.html @@ -24,22 +24,29 @@

プログラミング言語 Ruby リファレンスマニュアル

常に最新のバージョンのドキュメントを参照したい場合は上のリンク先が使えます。
- Ruby 2.7 - Ruby 2.6 - Ruby 2.5 + Ruby 3.4 + Ruby 3.3 + Ruby 3.2 + るりまサーチ (全文検索)

Other versions

Project Page

diff --git a/system/bc-setup-all b/system/bc-setup-all index f16ff8a..6a6d87e 100755 --- a/system/bc-setup-all +++ b/system/bc-setup-all @@ -3,10 +3,11 @@ require 'fileutils' VERSIONS = %w[ - 2.5.0 - 2.6.0 - 2.7.0 - 2.8.0 + 3.0 + 3.1 + 3.2 + 3.3 + 3.4 ] RUBY = "ruby" @@ -18,6 +19,8 @@ BITCLUST_LIB_DIR = "#{BITCLUST_BASE}/lib" DOC_BASE = "/var/rubydoc/doctree" REF_BASE = "refm" # was "#{DOC_BASE}/refm" +GENERATED_DOCUMENTS_BASE = "/var/rubydoc/generated-documents" + def setup_db(version) Dir.chdir(DOC_BASE) do db = "#{REF_BASE}/db-#{version}" @@ -37,6 +40,24 @@ if !File.directory?(DOC_BASE) system(*%W"git clone --depth 1 https://github.com/rurema/doctree.git #{DOC_BASE}") end +if !File.directory?(GENERATED_DOCUMENTS_BASE) + system(*%W"git clone --depth 1 https://github.com/rurema/generated-documents.git #{GENERATED_DOCUMENTS_BASE}") +end + +Dir.chdir(GENERATED_DOCUMENTS_BASE) do + system(*%w"git fetch origin") + system(*%w"git reset origin/main --hard") +end + +Dir.glob("#{GENERATED_DOCUMENTS_BASE}/db/db-*") do |db| + src = db + dst = "#{DOC_BASE}/#{REF_BASE}/#{File.basename(db)}" + FileUtils.rm_rf(dst) if File.directory?(dst) + system(*%W"ln -snf #{src} #{dst}") +end + +exit + Dir.chdir(BITCLUST_BASE) do system(*%w"git fetch origin") system(*%w"git reset origin/master --hard") diff --git a/system/bc-static-all b/system/bc-static-all index 006aae7..403f730 100755 --- a/system/bc-static-all +++ b/system/bc-static-all @@ -1,10 +1,11 @@ #!/usr/bin/env ruby VERSIONS = %w[ - 2.5.0 - 2.6.0 - 2.7.0 - 2.8.0 + 3.0 + 3.1 + 3.2 + 3.3 + 3.4 ] RUBY = "ruby" @@ -20,6 +21,14 @@ REF_BASE = "/var/rubydoc/doctree/refm" DOC_ROOT = "/var/www/docs.ruby-lang.org/shared/public/ja" +GENERATED_DOCUMENTS_BASE = "/var/rubydoc/generated-documents" + +Dir.glob("#{GENERATED_DOCUMENTS_BASE}/html/ja/*") do |version| + system("rsync", "-acvi", "--no-times", "--delete", version, DOC_ROOT) or raise +end + +exit + def create_document(version) system(RUBY, "-I#{BITCLUST_LIB_DIR}", BITCLUST, diff --git a/system/rdoc-static-all b/system/rdoc-static-all index d882f3d..dceaacb 100755 --- a/system/rdoc-static-all +++ b/system/rdoc-static-all @@ -1,9 +1,11 @@ #!/usr/bin/env ruby VERSIONS = %w[ - 2.5.0 - 2.6.0 - 2.7.0 + 3.0 + 3.1 + 3.2 + 3.3 + 3.4 master ] @@ -11,10 +13,11 @@ DOC_ROOT = ENV.fetch('DOC_ROOT', "/var/www/docs.ruby-lang.org/shared/public/en") Dir.chdir(File.join(__dir__, '..')) -def create_document(version) - system("bundle", "exec", "rake", "update:#{version}") or raise - system("bundle", "exec", "rake", "rdoc:clobber_en/#{version}") or raise - system("bundle", "exec", "rake", "rdoc:en/#{version}") or raise +def download_document(version) + system("rm", "-rf", version, "ruby-docs-en-#{version}.tar.xz") or raise + system("curl", "-sSLO", "https://cache.ruby-lang.org/pub/ruby/doc/ruby-docs-en-#{version}.tar.xz") or raise + system("ls", "-l", "ruby-docs-en-#{version}.tar.xz") + system("tar", "xf", "ruby-docs-en-#{version}.tar.xz") or raise system("rsync", "-acvi", "--no-times", "--delete", version, DOC_ROOT) or raise system("rm", "-rf", version) or raise system('./system/fastly-purge-key', '--soft', "en/#{version}") @@ -22,5 +25,5 @@ def create_document(version) end VERSIONS.reverse_each do |version| - create_document(version) + download_document(version) end diff --git a/system/ruby-lang.org.ru b/system/ruby-lang.org.ru index ae4e083..de2c45a 100644 --- a/system/ruby-lang.org.ru +++ b/system/ruby-lang.org.ru @@ -13,7 +13,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# along with this program. If not, see . Encoding.default_external = "utf-8"