From 9ba2e923f3aa5ef3c1a026bfcf4a2f4518a4e13d Mon Sep 17 00:00:00 2001 From: Cristian Maureira-Fredes Date: Sun, 11 Oct 2020 17:51:05 +0200 Subject: [PATCH 1/3] Agregar script para crear dict.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Usuarios en Windows, que no utilizan Git bash no pueden generar el archivo `dict.txt`, pues no tienen acceso al comando 'awk'. Si bien, la construcción de toda la documentación no es necesaria, este paso es importante incluso cuando se quiere hacer la verificación pospell a un archivo determinado, pues necesitamos el diccionario general que incluye todas las variaciones de 'dictionaries/' y 'dict'. --- .overrides/faq.rst | 4 ++-- .pre-commit-config.yaml | 4 ++-- .travis.yml | 2 +- Makefile | 4 +--- scripts/create_dict.py | 40 ++++++++++++++++++++++++++++++++++++++++ scripts/merge-dicts.sh | 2 -- 6 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 scripts/create_dict.py delete mode 100755 scripts/merge-dicts.sh diff --git a/.overrides/faq.rst b/.overrides/faq.rst index a816dbb1ed..78bb6e8acc 100644 --- a/.overrides/faq.rst +++ b/.overrides/faq.rst @@ -26,8 +26,8 @@ pospell. Pospell puede ser instalada en tu entorno de Python empleando pip Una vez instalado, para chequear el fichero .po sobre el que estás trabajando, ejecuta desde el directorio principal del repo:: - awk 1 dict dictionaries/*.txt > dict.txt - pospell -p dict.txt -l es_AR -l es_ES path/tu_fichero.po + python scripts/create_dict.py # para crear el archivo 'dict.txt' + pospell -p dict.txt -l es_ES path/tu_fichero.po pospell emplea la herramienta de diccionarios hunspell. Si pospell falla dando como error que no tiene hunspell instalado, lo puedes instalar así: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e12a13d92f..c7efaa4324 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,8 +7,8 @@ repos: hooks: - id: merge-dicts name: merge-dicts - entry: ./scripts/merge-dicts.sh - language: script + entry: python ./scripts/create_dict.py + language: python # This one requires package ``hunspell-es_es`` in Archlinux - repo: https://github.com/JulienPalard/pospell rev: v1.0.5 diff --git a/.travis.yml b/.travis.yml index d4b73e2897..caef7bfecb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: - powrap --version script: - powrap --check --quiet **/*.po - - awk 1 dict dictionaries/*.txt > dict.txt + - python scripts/create_dict.py - pospell -p dict.txt -l es_AR -l es_ES **/*.po - pip install -q -r requirements.txt - PYTHONWARNINGS=ignore::FutureWarning sphinx-build -j auto -W --keep-going -b html -d cpython/Doc/_build/doctree -D language=es . cpython/Doc/_build/html diff --git a/Makefile b/Makefile index af4788b25f..beda468e0e 100644 --- a/Makefile +++ b/Makefile @@ -89,9 +89,7 @@ progress: venv .PHONY: spell spell: venv - # 'cat' tenia el problema que algunos archivos no tenían una nueva línea al final - # 'awk 1' agregará una nueva línea en caso que falte. - awk 1 dict dictionaries/*.txt > dict.txt + $(VENV)/bin/python scripts/create_dict.py $(VENV)/bin/pospell -p dict.txt -l es_ES **/*.po diff --git a/scripts/create_dict.py b/scripts/create_dict.py new file mode 100644 index 0000000000..ac6848e60b --- /dev/null +++ b/scripts/create_dict.py @@ -0,0 +1,40 @@ +import os +import sys + +""" +Script to generate the 'dict.txt' dictionary based +on the custom dictionaries under the 'dictionaries/' directory, +but also considering the old words from the 'dict' file. + +This was done with: + awk 1 dict dictionaries/*.txt > dict.txt +but the problem was that windows users, not using Git bash +have the problem that 'awk' is not a valid command, so this +enable them to use the script instead. +""" + +entries = set() + +# Read custom dictionaries +for name in os.listdir("dictionaries"): + if name.endswith(".txt"): + filename = os.path.join("dictionaries", name) + with open(filename, "r") as f: + lines = [i.rstrip() for i in f.readlines()] + if lines: + entries.update(set(lines)) + del lines + +# Remove empty string, from empty lines +entries.remove("") + +# Read main 'dict' +with open("dict", "r") as f: + entries.update(set(i.rstrip() for i in f.readlines())) + +# Write the 'dict.txt' file +with open("dict.txt", "w") as f: + for e in entries: + f.write(e) + f.write("\n") +print("Created 'dict.txt'") diff --git a/scripts/merge-dicts.sh b/scripts/merge-dicts.sh deleted file mode 100755 index b96943aeff..0000000000 --- a/scripts/merge-dicts.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -awk 1 dict dictionaries/*.txt > dict.txt From 4a1d63b39039e8f6728273b74afdd094147c7e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristi=C3=A1n=20Maureira-Fredes?= Date: Sun, 11 Oct 2020 21:37:29 +0200 Subject: [PATCH 2/3] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Álvaro Mondéjar --- scripts/create_dict.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/create_dict.py b/scripts/create_dict.py index ac6848e60b..d240548c17 100644 --- a/scripts/create_dict.py +++ b/scripts/create_dict.py @@ -1,3 +1,4 @@ +import glob import os import sys @@ -16,14 +17,12 @@ entries = set() # Read custom dictionaries -for name in os.listdir("dictionaries"): - if name.endswith(".txt"): - filename = os.path.join("dictionaries", name) - with open(filename, "r") as f: - lines = [i.rstrip() for i in f.readlines()] - if lines: - entries.update(set(lines)) - del lines +for filename in glob.glob(os.path.join("dictionaries", "*.txt")): + with open(filename, "r") as f: + lines = [i.rstrip() for i in f.readlines()] + if lines: + entries.update(set(lines)) + del lines # Remove empty string, from empty lines entries.remove("") From 9fa6389b54301de84d2b6f694af222290afe7222 Mon Sep 17 00:00:00 2001 From: Cristian Maureira-Fredes Date: Sun, 11 Oct 2020 21:42:21 +0200 Subject: [PATCH 3/3] Use pathlib instead of os and glob --- scripts/create_dict.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/create_dict.py b/scripts/create_dict.py index d240548c17..882e92e106 100644 --- a/scripts/create_dict.py +++ b/scripts/create_dict.py @@ -1,6 +1,4 @@ -import glob -import os -import sys +from pathlib import Path """ Script to generate the 'dict.txt' dictionary based @@ -17,7 +15,7 @@ entries = set() # Read custom dictionaries -for filename in glob.glob(os.path.join("dictionaries", "*.txt")): +for filename in Path("dictionaries").glob("*.txt"): with open(filename, "r") as f: lines = [i.rstrip() for i in f.readlines()] if lines: