diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 05a68543b..24048b454 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -1,6 +1,6 @@ on: schedule: - - cron: '*/30 * * * *' + - cron: '17 * * * *' push: branches: - 'main' @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 - run: sudo apt-get install -y gettext - run: uv run generate.py # generates "index.html" - - run: mkdir -p build && cp index.* style.css build + - run: mkdir -p build && cp index.* style.css warnings* build - name: Deploy 🚀 if: github.event_name != 'pull_request' uses: JamesIves/github-pages-deploy-action@v4 diff --git a/.gitignore b/.gitignore index dcaf71693..aa69e2af2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ index.html +warnings-*.txt diff --git a/build_warnings.py b/build_warnings.py new file mode 100644 index 000000000..95faa092f --- /dev/null +++ b/build_warnings.py @@ -0,0 +1,37 @@ +from pathlib import Path +from re import findall +from shutil import copyfile + +import sphinx.cmd.build + + +def number(clones_dir: str, repo: str, language_code: str) -> int: + language_part, *locale = language_code.split('-') + if locale: + lang_with_locale = f'{language_part}_{locale[0].upper()}' + else: + lang_with_locale = language_part + locale_dir = Path(clones_dir, f'cpython/Doc/locales/{lang_with_locale}/LC_MESSAGES') + locale_dir.mkdir(parents=True) + for po_file in Path(clones_dir, repo).rglob('*.po'): + relative_path = po_file.relative_to(Path(clones_dir, repo)) + target_file = locale_dir / relative_path + target_file.parent.mkdir(parents=True, exist_ok=True) + copyfile(po_file, target_file) + sphinx.cmd.build.main( + ( + '--builder', + 'html', + '--jobs', + 'auto', + '--define', + f'language={language_code}', + '--verbose', + '--warning-file', + warning_file := f'{clones_dir}/warnings-{language_code}.txt', + f'{clones_dir}/cpython/Doc', # sourcedir + f'./sphinxbuild/{language_code}', # outputdir + ) + ) + copyfile(warning_file, f'warnings-{language_code}.txt') + return len(findall('ERROR|WARNING', Path(warning_file).read_text())) diff --git a/generate.py b/generate.py index a90cf1709..446967d2f 100644 --- a/generate.py +++ b/generate.py @@ -6,6 +6,8 @@ # "jinja2", # "requests", # "docutils", +# "sphinx", +# "python-docs-theme", # ] # /// import json @@ -23,11 +25,12 @@ from jinja2 import Template from urllib3 import PoolManager -import contribute +import build_warnings import build_status +import contribute from visitors import get_number_of_visitors from completion import branches_from_devguide, get_completion, TranslatorsData -from repositories import get_languages_and_repos, Language +from repositories import Language, get_languages_and_repos generation_time = datetime.now(timezone.utc) @@ -48,15 +51,13 @@ def get_completion_progress() -> Iterator['LanguageProjectData']: ) subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True) subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True) - languages_built = dict(build_status.get_languages(http := PoolManager())) - - with concurrent.futures.ThreadPoolExecutor() as executor: + languages_built = dict(build_status.get_languages(PoolManager())) + with concurrent.futures.ProcessPoolExecutor() as executor: return executor.map( get_project_data, *zip(*get_languages_and_repos(devguide_dir)), itertools.repeat(languages_built), itertools.repeat(clones_dir), - itertools.repeat(http), ) @@ -65,16 +66,21 @@ def get_project_data( repo: str | None, languages_built: dict[str, bool], clones_dir: str, - http: PoolManager, ) -> 'LanguageProjectData': built = language.code in languages_built if repo: completion, translators_data, branch = get_completion(clones_dir, repo) - visitors_num = get_number_of_visitors(language.code, http) if built else 0 + visitors_num = ( + get_number_of_visitors(language.code, PoolManager()) if built else 0 + ) + warnings = ( + build_warnings.number(clones_dir, repo, language.code) if completion else 0 + ) else: completion = 0.0 translators_data = TranslatorsData(0, False) visitors_num = 0 + warnings = 0 branch = None return LanguageProjectData( language, @@ -83,6 +89,7 @@ def get_project_data( completion, translators_data, visitors_num, + warnings, built, in_switcher=languages_built.get(language.code), uses_platform=language.code in contribute.pulling_from_transifex, @@ -98,6 +105,7 @@ class LanguageProjectData: completion: float translators: TranslatorsData visitors: int + warnings: int built: bool in_switcher: bool | None uses_platform: bool diff --git a/style.css b/style.css index 3a67a64ad..c4fbf443f 100644 --- a/style.css +++ b/style.css @@ -39,7 +39,7 @@ th { .progress-bar.low + .progress-bar-outer-label { display: inline-block; } -td[data-label="visitors"], td[data-label="translators"] { +td[data-label="visitors"], td[data-label="translators"], td[data-label="warnings"] { text-align: right; } td[data-label="completion"] { diff --git a/template.html.jinja b/template.html.jinja index 1fefcff66..063d27838 100644 --- a/template.html.jinja +++ b/template.html.jinja @@ -3,6 +3,7 @@ Python Docs Translation Dashboard +

Python Docs Translation Dashboard

@@ -12,8 +13,9 @@ language contribute build - visitors* + visitors¹ translators + warnings² completion @@ -22,20 +24,20 @@ {{ project.language.name }} ({{ project.language.code }}) - {% if project.contribution_link %}{% endif %} + {% if project.contribution_link %}{% endif %} {% if project.uses_platform %}platform{% else %}repository{% endif %} {% if project.contribution_link %}{% endif %} {% if project.built %} - ✓{% if project.in_switcher %} in switcher{% endif %} + ✓{% if project.in_switcher %} in switcher{% endif %} {% else %} ✗ {% endif %} {% if project.built %} - + {{ '{:,}'.format(project.visitors) }} {% else %} @@ -43,10 +45,13 @@ {% endif %} - {% if project.translators.link %}{% endif %} + {% if project.translators.link %}{% endif %} {{ project.translators.number }} {% if project.translators.link %}{% endif %} + + {% if project.completion %}{{ project.warnings }}{% else %}{{ project.warnings }}{% endif %} +
{{ "{:.2f}".format(project.completion) }}%
{{ "{:.2f}".format(project.completion) }}%
@@ -55,8 +60,9 @@ {% endfor %} -

* sum of daily unique visitors since 8 June 2024

-

For more information about translations, see the Python Developer’s Guide.

+

¹ sum of daily unique visitors since 8 June 2024

+

² number of Sphinx build process warnings

+

For more information about translations, see the Python Developer’s Guide.

Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).