Skip to content

gh-130197: Fix (and test) option in pygettext #133021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions Lib/test/test_tools/i18n_data/messages.pot
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ msgid ""
" multiline!\n"
msgstr ""

#: messages.py:46 messages.py:89 messages.py:90 messages.py:93 messages.py:94
#: messages.py:99 messages.py:100 messages.py:101
#: messages.py:46 messages.py:92 messages.py:93 messages.py:96 messages.py:97
#: messages.py:102 messages.py:103 messages.py:104
msgid "foo"
msgid_plural "foos"
msgstr[0] ""
Expand Down Expand Up @@ -80,18 +80,18 @@ msgstr ""
msgid "default value"
msgstr ""

#: messages.py:91 messages.py:92 messages.py:95 messages.py:96
#: messages.py:94 messages.py:95 messages.py:98 messages.py:99
msgctxt "context"
msgid "foo"
msgid_plural "foos"
msgstr[0] ""
msgstr[1] ""

#: messages.py:102
#: messages.py:105
msgid "domain foo"
msgstr ""

#: messages.py:118 messages.py:119
#: messages.py:121 messages.py:122
msgid "world"
msgid_plural "worlds"
msgstr[0] ""
Expand Down
3 changes: 3 additions & 0 deletions Lib/test/test_tools/i18n_data/messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ def _(x="don't extract me"):
pass


def func():
"""Docstring..."""

# Other gettext functions
gettext("foo")
ngettext("foo", "foos", 1)
Expand Down
9 changes: 9 additions & 0 deletions Lib/test/test_tools/i18n_data/skipdocstrings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
def test(x):
"""'I think he’s had a heart attack!'
'IF THERE’S ONE THING I CAN’T STAND IS PEOPLE HAVING HEART ATTACKS!!!'
'I’m fine, now!'
'Good. Now we’re getting somewhere.'
"""

class Foo:
"""Docstring"""
6 changes: 6 additions & 0 deletions Lib/test/test_tools/i18n_data/skipdocstrings.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
fileloc.py

skipdocstrings.py
messages.py
messages.py
test*.py
2 changes: 2 additions & 0 deletions Lib/test/test_tools/i18n_data/testfile.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def func():
"""Get the Holy Hand Grenade!"""
20 changes: 12 additions & 8 deletions Lib/test/test_tools/test_i18n.py
Original file line number Diff line number Diff line change
Expand Up @@ -589,6 +589,8 @@ def extract_from_snapshots():
'messages.py': (),
'fileloc.py': ('--docstrings',),
'docstrings.py': ('--docstrings',),
('docstrings.py', 'skipdocstrings.py', 'testfile.py', 'docstrings.pot'): ('--docstrings',
f'--exclude-docstrings={DATA_DIR}{os.path.sep}skipdocstrings.txt'),
'comments.py': ('--add-comments=i18n:',),
'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2',
'--keyword=pfoo:1c,2',
Expand All @@ -606,18 +608,20 @@ def extract_from_snapshots():

for filename, args in snapshots.items():
if isinstance(filename, tuple):
filename, output_file = filename
*filenames, output_file = filename
output_file = DATA_DIR / output_file
input_file = DATA_DIR / filename
input_files = [DATA_DIR / file for file in filenames]
else:
input_file = DATA_DIR / filename
output_file = input_file.with_suffix('.pot')
contents = input_file.read_bytes()
input_files = [DATA_DIR / filename]
output_file = input_files[0].with_suffix('.pot')

with temp_cwd(None):
Path(input_file.name).write_bytes(contents)
for input_file in input_files:
contents = input_file.read_bytes()
Path(input_file.name).write_bytes(contents)
assert_python_ok('-Xutf8', Test_pygettext.script, *args,
input_file.name)
yield (input_file, output_file,
*[file.name for file in input_files])
yield (input_files, output_file,
Path('messages.pot').read_text(encoding='utf-8'))


Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix and test the ``--exclude-docstrings`` option in :program:`pygettext`.
Note: the option was also renamed from ``--no-docstrings`` for clarity.
39 changes: 19 additions & 20 deletions Tools/i18n/pygettext.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@
-o filename
--output=filename
Rename the default output file from messages.pot to filename. If
filename is `-' then the output is sent to standard out.
filename is '-' then the output is sent to standard out.

-p dir
--output-dir=dir
Expand All @@ -110,7 +110,7 @@
Solaris # File: filename, line: line-number
GNU #: filename:line

The style name is case insensitive. GNU style is the default.
The style name is case-insensitive. GNU style is the default.

-v
--verbose
Expand All @@ -131,15 +131,18 @@
appear on a line by itself in the file.

-X filename
--no-docstrings=filename
--exclude-docstrings=
This is only useful in conjunction with the -D option above.
Specify a file that contains a list of files (one per line) that
should not have their docstrings extracted. This is only useful in
conjunction with the -D option above.
should not have their docstrings extracted. fnmatch-style patterns are
supported.

If `inputfile' is -, standard input is read.

If 'inputfile' is -, standard input is read.
"""

import ast
import fnmatch
import getopt
import glob
import importlib.machinery
Expand Down Expand Up @@ -188,7 +191,7 @@ def make_escapes(pass_nonascii):
global escapes, escape
if pass_nonascii:
# Allow non-ascii characters to pass through so that e.g. 'msgid
# "Höhe"' would not result in 'msgid "H\366he"'. Otherwise we
# "Höhe"' would not result in 'msgid "H\366he"'. Otherwise, we
# escape any character outside the 32..126 range.
escape = escape_ascii
else:
Expand Down Expand Up @@ -483,7 +486,8 @@ def visit_Call(self, node):

def _extract_docstring(self, node):
if (not self.options.docstrings or
self.options.nodocstrings.get(self.filename)):
any(fnmatch.fnmatch(self.filename, pattern)
for pattern in self.options.nodocstrings)):
return

docstring = ast.get_docstring(node)
Expand Down Expand Up @@ -692,7 +696,7 @@ def main():
'help', 'keyword=', 'no-default-keywords',
'add-location', 'no-location', 'output=', 'output-dir=',
'style=', 'verbose', 'version', 'width=', 'exclude-file=',
'docstrings', 'no-docstrings',
'docstrings', 'exclude-docstrings=',
])
except getopt.error as msg:
usage(1, msg)
Expand All @@ -714,7 +718,7 @@ class Options:
width = 78
excludefilename = ''
docstrings = 0
nodocstrings = {}
nodocstrings = set()
comment_tags = set()

options = Options()
Expand Down Expand Up @@ -766,16 +770,11 @@ class Options:
usage(1, f'--width argument must be an integer: {arg}')
elif opt in ('-x', '--exclude-file'):
options.excludefilename = arg
elif opt in ('-X', '--no-docstrings'):
fp = open(arg)
try:
while 1:
line = fp.readline()
if not line:
break
options.nodocstrings[line[:-1]] = 1
finally:
fp.close()
elif opt in ('-X', '--exclude-docstrings'):
with open(arg) as nodocstrings_file:
for line in nodocstrings_file:
line = line.strip()
options.nodocstrings.add(line)

options.comment_tags = tuple(options.comment_tags)

Expand Down
Loading