From f4db75c056bda170be5652a3daa9964b6151def8 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 1 Feb 2025 01:21:47 +0100 Subject: [PATCH 01/21] Track number of Sphinx build warnings --- .github/workflows/update.yml | 2 +- build_warnings.py | 29 +++++++++++++++++++++++++++++ generate.py | 11 +++++++++-- style.css | 2 +- template.html.jinja | 7 +++++-- 5 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 build_warnings.py diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 23259e2fe..5f1cde312 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v4 - run: sudo apt-get install -y gettext - run: uv run generate.py # generates "index.html" - - run: mkdir -p build && cp index.html style.css build + - run: mkdir -p build && cp index.html style.css warnings* build - name: Deploy 🚀 if: github.event_name != 'pull_request' uses: JamesIves/github-pages-deploy-action@v4 diff --git a/build_warnings.py b/build_warnings.py new file mode 100644 index 000000000..da0e6de15 --- /dev/null +++ b/build_warnings.py @@ -0,0 +1,29 @@ +from pathlib import Path +from shutil import copyfile + +import sphinx.cmd.build + + +def number(clones_dir: str, repo: str, language_code: str) -> int: + Path(clones_dir, f'cpython/Doc/locales/{language_code}').mkdir(parents=True) + Path(clones_dir, f'cpython/Doc/locales/{language_code}/LC_MESSAGES').symlink_to( + Path(clones_dir, repo), True + ) + sphinx.cmd.build.main( + ( + '--builder', + 'html', + '--fresh-env', + '--jobs', + 'auto', + '--define', + f'language={language_code}', + '--verbose', + '--warning-file', + warning_file := f'{clones_dir}/warnings-{language_code}.txt', + f'{clones_dir}/cpython/Doc', # sourcedir + './build', # outputdir + ) + ) + copyfile(warning_file, f'warnings-{language_code}.txt') + return len(Path(warning_file).read_text().splitlines()) diff --git a/generate.py b/generate.py index 237e12f7e..dae8ed342 100644 --- a/generate.py +++ b/generate.py @@ -6,6 +6,8 @@ # "jinja2", # "requests", # "docutils", +# "sphinx", +# "python-docs-theme", # ] # /// import subprocess @@ -19,11 +21,12 @@ from git import Repo from jinja2 import Template -import contribute +import build_warnings import build_status +import contribute from visitors import get_number_of_visitors from completion import branches_from_devguide, get_completion, TranslatorsData -from repositories import get_languages_and_repos, Language +from repositories import Language, get_languages_and_repos generation_time = datetime.now(timezone.utc) @@ -50,16 +53,19 @@ def get_completion_progress() -> Iterator['LanguageProjectData']: if repo: completion, translators_data = get_completion(clones_dir, repo) visitors_num = get_number_of_visitors(language.code) if built else 0 + issues = build_warnings.number(clones_dir, repo, language.code) else: completion = 0.0 translators_data = TranslatorsData(0, False) visitors_num = 0 + issues = 0 yield LanguageProjectData( language, repo, completion, translators_data, visitors_num, + issues, built, in_switcher=languages_built.get(language.code), uses_platform=language.code in contribute.pulling_from_transifex, @@ -74,6 +80,7 @@ class LanguageProjectData: completion: float translators: TranslatorsData visitors: int + issues: int built: bool in_switcher: bool | None uses_platform: bool diff --git a/style.css b/style.css index 3a67a64ad..f55a59ed8 100644 --- a/style.css +++ b/style.css @@ -39,7 +39,7 @@ th { .progress-bar.low + .progress-bar-outer-label { display: inline-block; } -td[data-label="visitors"], td[data-label="translators"] { +td[data-label="visitors"], td[data-label="translators"], td[data-label="issues"] { text-align: right; } td[data-label="completion"] { diff --git a/template.html.jinja b/template.html.jinja index 1fefcff66..199f7d1d8 100644 --- a/template.html.jinja +++ b/template.html.jinja @@ -12,8 +12,9 @@ language contribute build - visitors* + visitors¹ translators + warnings² completion @@ -47,6 +48,7 @@ {{ project.translators.number }} {% if project.translators.link %}{% endif %} + {{ project.issues }}
{{ "{:.2f}".format(project.completion) }}%
{{ "{:.2f}".format(project.completion) }}%
@@ -55,7 +57,8 @@ {% endfor %} -

* sum of daily unique visitors since 8 June 2024

+

¹ sum of daily unique visitors since 8 June 2024

+

² number of Sphinx build process warnings

For more information about translations, see the Python Developer’s Guide.

Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).

From 84e0a79d13ddf4f03f39ef092f0f6317870f3275 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 1 Feb 2025 02:38:02 +0100 Subject: [PATCH 02/21] Rename Sphinx outputdir, build is used to copy files for GH pages copy --- build_warnings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_warnings.py b/build_warnings.py index da0e6de15..f73f8922c 100644 --- a/build_warnings.py +++ b/build_warnings.py @@ -22,7 +22,7 @@ def number(clones_dir: str, repo: str, language_code: str) -> int: '--warning-file', warning_file := f'{clones_dir}/warnings-{language_code}.txt', f'{clones_dir}/cpython/Doc', # sourcedir - './build', # outputdir + './sphinxbuild', # outputdir ) ) copyfile(warning_file, f'warnings-{language_code}.txt') From 07618728144bb241e2927e6fc07059600db2ade4 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 1 Feb 2025 02:49:58 +0100 Subject: [PATCH 03/21] Copy over only PO files to avoid warnings on extra files --- build_warnings.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build_warnings.py b/build_warnings.py index f73f8922c..7785c6ecc 100644 --- a/build_warnings.py +++ b/build_warnings.py @@ -5,10 +5,13 @@ def number(clones_dir: str, repo: str, language_code: str) -> int: - Path(clones_dir, f'cpython/Doc/locales/{language_code}').mkdir(parents=True) - Path(clones_dir, f'cpython/Doc/locales/{language_code}/LC_MESSAGES').symlink_to( - Path(clones_dir, repo), True - ) + locale_dir = Path(clones_dir, f'cpython/Doc/locales/{language_code}/LC_MESSAGES') + locale_dir.mkdir(parents=True) + for po_file in Path(clones_dir, repo).rglob('*.po'): + relative_path = po_file.relative_to(Path(clones_dir, repo)) + target_file = locale_dir / relative_path + target_file.parent.mkdir(parents=True, exist_ok=True) + copyfile(po_file, target_file) sphinx.cmd.build.main( ( '--builder', From e72211602f8c5a6ad508f3e09ee0ebcc5bdf45b5 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 1 Feb 2025 02:51:42 +0100 Subject: [PATCH 04/21] Use locale format for Sphinx --- build_warnings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build_warnings.py b/build_warnings.py index 7785c6ecc..624db1599 100644 --- a/build_warnings.py +++ b/build_warnings.py @@ -5,7 +5,12 @@ def number(clones_dir: str, repo: str, language_code: str) -> int: - locale_dir = Path(clones_dir, f'cpython/Doc/locales/{language_code}/LC_MESSAGES') + language_part, *locale = language_code.split('-') + if locale: + lang_with_locale = f'{language_part}_{locale[0].upper()}' + else: + lang_with_locale = language_part + locale_dir = Path(clones_dir, f'cpython/Doc/locales/{lang_with_locale}/LC_MESSAGES') locale_dir.mkdir(parents=True) for po_file in Path(clones_dir, repo).rglob('*.po'): relative_path = po_file.relative_to(Path(clones_dir, repo)) From bd676c0196031a367331111d64588917e2acc8fd Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 1 Feb 2025 02:53:52 +0100 Subject: [PATCH 05/21] Don't override language builds --- build_warnings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_warnings.py b/build_warnings.py index 624db1599..f268d71c6 100644 --- a/build_warnings.py +++ b/build_warnings.py @@ -30,7 +30,7 @@ def number(clones_dir: str, repo: str, language_code: str) -> int: '--warning-file', warning_file := f'{clones_dir}/warnings-{language_code}.txt', f'{clones_dir}/cpython/Doc', # sourcedir - './sphinxbuild', # outputdir + f'./sphinxbuild/{language_code}', # outputdir ) ) copyfile(warning_file, f'warnings-{language_code}.txt') From add0365fd3f97150b553b99b6a0a28d2f42ab9c8 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 1 Feb 2025 12:31:22 +0100 Subject: [PATCH 06/21] Drop fresh-env flag from Sphinx build --- build_warnings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build_warnings.py b/build_warnings.py index f268d71c6..900b10a9a 100644 --- a/build_warnings.py +++ b/build_warnings.py @@ -21,7 +21,6 @@ def number(clones_dir: str, repo: str, language_code: str) -> int: ( '--builder', 'html', - '--fresh-env', '--jobs', 'auto', '--define', From f97278d8583716a9e37fdf6059a84212cea5434e Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 1 Feb 2025 13:19:06 +0100 Subject: [PATCH 07/21] Run update on main once an hour Build now lasts for ~40 minutes --- .github/workflows/update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 5f1cde312..1c7070f08 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -1,6 +1,6 @@ on: schedule: - - cron: '*/30 * * * *' + - cron: '17 * * * *' push: branches: - 'main' From 064faa25b7085379520ff6159f1482944fa1b76b Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 3 Feb 2025 00:35:41 +0100 Subject: [PATCH 08/21] Parallelize the main loop --- generate.py | 57 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/generate.py b/generate.py index dae8ed342..223a1c1cc 100644 --- a/generate.py +++ b/generate.py @@ -12,8 +12,10 @@ # /// import subprocess from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime, timezone +from itertools import repeat from logging import info from pathlib import Path from tempfile import TemporaryDirectory @@ -48,31 +50,42 @@ def get_completion_progress() -> Iterator['LanguageProjectData']: subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True) subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True) languages_built = dict(build_status.get_languages()) - for language, repo in get_languages_and_repos(devguide_dir): - built = language.code in languages_built - if repo: - completion, translators_data = get_completion(clones_dir, repo) - visitors_num = get_number_of_visitors(language.code) if built else 0 - issues = build_warnings.number(clones_dir, repo, language.code) - else: - completion = 0.0 - translators_data = TranslatorsData(0, False) - visitors_num = 0 - issues = 0 - yield LanguageProjectData( - language, - repo, - completion, - translators_data, - visitors_num, - issues, - built, - in_switcher=languages_built.get(language.code), - uses_platform=language.code in contribute.pulling_from_transifex, - contribution_link=contribute.get_contrib_link(language.code, repo), + with ThreadPoolExecutor() as executor: + return executor.map( + get_data, + *zip(*get_languages_and_repos(devguide_dir)), + repeat(languages_built), + repeat(clones_dir), ) +def get_data( + language: Language, repo: str, languages_built: dict[str, bool], clones_dir: str +) -> 'LanguageProjectData': + built = language.code in languages_built + if repo: + completion, translators_data = get_completion(clones_dir, repo) + visitors_num = get_number_of_visitors(language.code) if built else 0 + issues = build_warnings.number(clones_dir, repo, language.code) + else: + completion = 0.0 + translators_data = TranslatorsData(0, False) + visitors_num = 0 + issues = 0 + return LanguageProjectData( + language, + repo, + completion, + translators_data, + visitors_num, + issues, + built, + in_switcher=languages_built.get(language.code), + uses_platform=language.code in contribute.pulling_from_transifex, + contribution_link=contribute.get_contrib_link(language.code, repo), + ) + + @dataclass(frozen=True) class LanguageProjectData: language: Language From 82d6f0360aa994288dc1266a8922f2a258d116d2 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 3 Feb 2025 00:37:21 +0100 Subject: [PATCH 09/21] Rename issues to warnings --- generate.py | 8 ++++---- template.html.jinja | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/generate.py b/generate.py index 223a1c1cc..8c7eabff5 100644 --- a/generate.py +++ b/generate.py @@ -66,19 +66,19 @@ def get_data( if repo: completion, translators_data = get_completion(clones_dir, repo) visitors_num = get_number_of_visitors(language.code) if built else 0 - issues = build_warnings.number(clones_dir, repo, language.code) + warnings = build_warnings.number(clones_dir, repo, language.code) else: completion = 0.0 translators_data = TranslatorsData(0, False) visitors_num = 0 - issues = 0 + warnings = 0 return LanguageProjectData( language, repo, completion, translators_data, visitors_num, - issues, + warnings, built, in_switcher=languages_built.get(language.code), uses_platform=language.code in contribute.pulling_from_transifex, @@ -93,7 +93,7 @@ class LanguageProjectData: completion: float translators: TranslatorsData visitors: int - issues: int + warnings: int built: bool in_switcher: bool | None uses_platform: bool diff --git a/template.html.jinja b/template.html.jinja index 199f7d1d8..c32d5905c 100644 --- a/template.html.jinja +++ b/template.html.jinja @@ -48,7 +48,7 @@ {{ project.translators.number }} {% if project.translators.link %}{% endif %} - {{ project.issues }} + {{ project.warnings }}
{{ "{:.2f}".format(project.completion) }}%
{{ "{:.2f}".format(project.completion) }}%
From 6f565fe352a704e7c522d5337ded46bd17a17eb5 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 3 Feb 2025 00:39:47 +0100 Subject: [PATCH 10/21] Ignore generated warnings files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index dcaf71693..aa69e2af2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ index.html +warnings-*.txt From 562cc3d733c6aba49a98227865ab7be99cb44d73 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 3 Feb 2025 00:40:28 +0100 Subject: [PATCH 11/21] Rename issues to warnings in style.css --- style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/style.css b/style.css index f55a59ed8..c4fbf443f 100644 --- a/style.css +++ b/style.css @@ -39,7 +39,7 @@ th { .progress-bar.low + .progress-bar-outer-label { display: inline-block; } -td[data-label="visitors"], td[data-label="translators"], td[data-label="issues"] { +td[data-label="visitors"], td[data-label="translators"], td[data-label="warnings"] { text-align: right; } td[data-label="completion"] { From 77acb8bc47ae8dbd42aa1dcb4b78e306f0b1f402 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 3 Feb 2025 00:54:37 +0100 Subject: [PATCH 12/21] Skip build for zero completion --- generate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/generate.py b/generate.py index 8c7eabff5..3bf8f3a55 100644 --- a/generate.py +++ b/generate.py @@ -66,7 +66,9 @@ def get_data( if repo: completion, translators_data = get_completion(clones_dir, repo) visitors_num = get_number_of_visitors(language.code) if built else 0 - warnings = build_warnings.number(clones_dir, repo, language.code) + warnings = ( + build_warnings.number(clones_dir, repo, language.code) if completion else 0 + ) else: completion = 0.0 translators_data = TranslatorsData(0, False) From b99fd95aed16cec8a5668132ee19847c1fb80193 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 3 Feb 2025 01:14:04 +0100 Subject: [PATCH 13/21] Switch to ProcessPoolExecutor Sphinx html build writing seems not to be thread-safe --- generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate.py b/generate.py index 3bf8f3a55..9b3984840 100644 --- a/generate.py +++ b/generate.py @@ -12,7 +12,7 @@ # /// import subprocess from collections.abc import Iterator -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ProcessPoolExecutor from dataclasses import dataclass from datetime import datetime, timezone from itertools import repeat @@ -50,7 +50,7 @@ def get_completion_progress() -> Iterator['LanguageProjectData']: subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True) subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True) languages_built = dict(build_status.get_languages()) - with ThreadPoolExecutor() as executor: + with ProcessPoolExecutor() as executor: return executor.map( get_data, *zip(*get_languages_and_repos(devguide_dir)), From d515419d8e93b2383598cc508861fc9de1757719 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 3 Feb 2025 02:03:33 +0100 Subject: [PATCH 14/21] Use HTML base tag in head to specify default anchors target --- template.html.jinja | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/template.html.jinja b/template.html.jinja index c32d5905c..d5328231e 100644 --- a/template.html.jinja +++ b/template.html.jinja @@ -3,6 +3,7 @@ Python Docs Translation Dashboard +

Python Docs Translation Dashboard

@@ -23,20 +24,20 @@ {{ project.language.name }} ({{ project.language.code }}) - {% if project.contribution_link %}{% endif %} + {% if project.contribution_link %}{% endif %} {% if project.uses_platform %}platform{% else %}repository{% endif %} {% if project.contribution_link %}{% endif %} {% if project.built %} - ✓{% if project.in_switcher %} in switcher{% endif %} + ✓{% if project.in_switcher %} in switcher{% endif %} {% else %} ✗ {% endif %} {% if project.built %} - + {{ '{:,}'.format(project.visitors) }} {% else %} @@ -44,11 +45,11 @@ {% endif %} - {% if project.translators.link %}{% endif %} + {% if project.translators.link %}{% endif %} {{ project.translators.number }} {% if project.translators.link %}{% endif %} - {{ project.warnings }} + {{ project.warnings }}
{{ "{:.2f}".format(project.completion) }}%
{{ "{:.2f}".format(project.completion) }}%
@@ -57,9 +58,9 @@ {% endfor %} -

¹ sum of daily unique visitors since 8 June 2024

+

¹ sum of daily unique visitors since 8 June 2024

² number of Sphinx build process warnings

-

For more information about translations, see the Python Developer’s Guide.

+

For more information about translations, see the Python Developer’s Guide.

Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).