Skip to content

Commit e4cfdaa

Browse files
miss-islingtonronaldoussoren
miss-islington
and
ronaldoussoren
authored
GH-87235: Make sure "python /dev/fd/9 9</path/to/script.py" works on macOS (GH-99768)
On macOS all file descriptors for a particular file in /dev/fd share the same file offset, that is ``open("/dev/fd/9", "r")`` behaves more like ``dup(9)`` than a regular open. This causes problems when a user tries to run "/dev/fd/9" as a script because zipimport changes the file offset to try to read a zipfile directory. Therefore change zipimport to reset the file offset after trying to read the zipfile directory. (cherry picked from commit d08fb25) Co-authored-by: Ronald Oussoren <ronaldoussoren@mac.com>
1 parent 8bb7fda commit e4cfdaa

File tree

3 files changed

+120
-98
lines changed

3 files changed

+120
-98
lines changed

‎Lib/test/test_cmd_line_script.py

+14
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,20 @@ def test_nonexisting_script(self):
740740
self.assertIn(": can't open file ", err)
741741
self.assertNotEqual(proc.returncode, 0)
742742

743+
@unittest.skipUnless(os.path.exists('/dev/fd/0'), 'requires /dev/fd platform')
744+
def test_script_as_dev_fd(self):
745+
# GH-87235: On macOS passing a non-trivial script to /dev/fd/N can cause
746+
# problems because all open /dev/fd/N file descriptors share the same
747+
# offset.
748+
script = 'print("12345678912345678912345")'
749+
with os_helper.temp_dir() as work_dir:
750+
script_name = _make_test_script(work_dir, 'script.py', script)
751+
with open(script_name, "r") as fp:
752+
p = spawn_python(f"/dev/fd/{fp.fileno()}", close_fds=False, pass_fds=(0,1,2,fp.fileno()))
753+
out, err = p.communicate()
754+
self.assertEqual(out, b"12345678912345678912345\n")
755+
756+
743757

744758
def tearDownModule():
745759
support.reap_children()

‎Lib/zipimport.py

+105-98
Original file line numberDiff line numberDiff line change
@@ -405,114 +405,121 @@ def _read_directory(archive):
405405
raise ZipImportError(f"can't open Zip file: {archive!r}", path=archive)
406406

407407
with fp:
408+
# GH-87235: On macOS all file descriptors for /dev/fd/N share the same
409+
# file offset, reset the file offset after scanning the zipfile diretory
410+
# to not cause problems when some runs 'python3 /dev/fd/9 9<some_script'
411+
start_offset = fp.tell()
408412
try:
409-
fp.seek(-END_CENTRAL_DIR_SIZE, 2)
410-
header_position = fp.tell()
411-
buffer = fp.read(END_CENTRAL_DIR_SIZE)
412-
except OSError:
413-
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
414-
if len(buffer) != END_CENTRAL_DIR_SIZE:
415-
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
416-
if buffer[:4] != STRING_END_ARCHIVE:
417-
# Bad: End of Central Dir signature
418-
# Check if there's a comment.
419413
try:
420-
fp.seek(0, 2)
421-
file_size = fp.tell()
422-
except OSError:
423-
raise ZipImportError(f"can't read Zip file: {archive!r}",
424-
path=archive)
425-
max_comment_start = max(file_size - MAX_COMMENT_LEN -
426-
END_CENTRAL_DIR_SIZE, 0)
427-
try:
428-
fp.seek(max_comment_start)
429-
data = fp.read()
430-
except OSError:
431-
raise ZipImportError(f"can't read Zip file: {archive!r}",
432-
path=archive)
433-
pos = data.rfind(STRING_END_ARCHIVE)
434-
if pos < 0:
435-
raise ZipImportError(f'not a Zip file: {archive!r}',
436-
path=archive)
437-
buffer = data[pos:pos+END_CENTRAL_DIR_SIZE]
438-
if len(buffer) != END_CENTRAL_DIR_SIZE:
439-
raise ZipImportError(f"corrupt Zip file: {archive!r}",
440-
path=archive)
441-
header_position = file_size - len(data) + pos
442-
443-
header_size = _unpack_uint32(buffer[12:16])
444-
header_offset = _unpack_uint32(buffer[16:20])
445-
if header_position < header_size:
446-
raise ZipImportError(f'bad central directory size: {archive!r}', path=archive)
447-
if header_position < header_offset:
448-
raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive)
449-
header_position -= header_size
450-
arc_offset = header_position - header_offset
451-
if arc_offset < 0:
452-
raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive)
453-
454-
files = {}
455-
# Start of Central Directory
456-
count = 0
457-
try:
458-
fp.seek(header_position)
459-
except OSError:
460-
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
461-
while True:
462-
buffer = fp.read(46)
463-
if len(buffer) < 4:
464-
raise EOFError('EOF read where not expected')
465-
# Start of file header
466-
if buffer[:4] != b'PK\x01\x02':
467-
break # Bad: Central Dir File Header
468-
if len(buffer) != 46:
469-
raise EOFError('EOF read where not expected')
470-
flags = _unpack_uint16(buffer[8:10])
471-
compress = _unpack_uint16(buffer[10:12])
472-
time = _unpack_uint16(buffer[12:14])
473-
date = _unpack_uint16(buffer[14:16])
474-
crc = _unpack_uint32(buffer[16:20])
475-
data_size = _unpack_uint32(buffer[20:24])
476-
file_size = _unpack_uint32(buffer[24:28])
477-
name_size = _unpack_uint16(buffer[28:30])
478-
extra_size = _unpack_uint16(buffer[30:32])
479-
comment_size = _unpack_uint16(buffer[32:34])
480-
file_offset = _unpack_uint32(buffer[42:46])
481-
header_size = name_size + extra_size + comment_size
482-
if file_offset > header_offset:
483-
raise ZipImportError(f'bad local header offset: {archive!r}', path=archive)
484-
file_offset += arc_offset
485-
486-
try:
487-
name = fp.read(name_size)
414+
fp.seek(-END_CENTRAL_DIR_SIZE, 2)
415+
header_position = fp.tell()
416+
buffer = fp.read(END_CENTRAL_DIR_SIZE)
488417
except OSError:
489418
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
490-
if len(name) != name_size:
419+
if len(buffer) != END_CENTRAL_DIR_SIZE:
491420
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
492-
# On Windows, calling fseek to skip over the fields we don't use is
493-
# slower than reading the data because fseek flushes stdio's
494-
# internal buffers. See issue #8745.
421+
if buffer[:4] != STRING_END_ARCHIVE:
422+
# Bad: End of Central Dir signature
423+
# Check if there's a comment.
424+
try:
425+
fp.seek(0, 2)
426+
file_size = fp.tell()
427+
except OSError:
428+
raise ZipImportError(f"can't read Zip file: {archive!r}",
429+
path=archive)
430+
max_comment_start = max(file_size - MAX_COMMENT_LEN -
431+
END_CENTRAL_DIR_SIZE, 0)
432+
try:
433+
fp.seek(max_comment_start)
434+
data = fp.read()
435+
except OSError:
436+
raise ZipImportError(f"can't read Zip file: {archive!r}",
437+
path=archive)
438+
pos = data.rfind(STRING_END_ARCHIVE)
439+
if pos < 0:
440+
raise ZipImportError(f'not a Zip file: {archive!r}',
441+
path=archive)
442+
buffer = data[pos:pos+END_CENTRAL_DIR_SIZE]
443+
if len(buffer) != END_CENTRAL_DIR_SIZE:
444+
raise ZipImportError(f"corrupt Zip file: {archive!r}",
445+
path=archive)
446+
header_position = file_size - len(data) + pos
447+
448+
header_size = _unpack_uint32(buffer[12:16])
449+
header_offset = _unpack_uint32(buffer[16:20])
450+
if header_position < header_size:
451+
raise ZipImportError(f'bad central directory size: {archive!r}', path=archive)
452+
if header_position < header_offset:
453+
raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive)
454+
header_position -= header_size
455+
arc_offset = header_position - header_offset
456+
if arc_offset < 0:
457+
raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive)
458+
459+
files = {}
460+
# Start of Central Directory
461+
count = 0
495462
try:
496-
if len(fp.read(header_size - name_size)) != header_size - name_size:
497-
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
463+
fp.seek(header_position)
498464
except OSError:
499465
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
466+
while True:
467+
buffer = fp.read(46)
468+
if len(buffer) < 4:
469+
raise EOFError('EOF read where not expected')
470+
# Start of file header
471+
if buffer[:4] != b'PK\x01\x02':
472+
break # Bad: Central Dir File Header
473+
if len(buffer) != 46:
474+
raise EOFError('EOF read where not expected')
475+
flags = _unpack_uint16(buffer[8:10])
476+
compress = _unpack_uint16(buffer[10:12])
477+
time = _unpack_uint16(buffer[12:14])
478+
date = _unpack_uint16(buffer[14:16])
479+
crc = _unpack_uint32(buffer[16:20])
480+
data_size = _unpack_uint32(buffer[20:24])
481+
file_size = _unpack_uint32(buffer[24:28])
482+
name_size = _unpack_uint16(buffer[28:30])
483+
extra_size = _unpack_uint16(buffer[30:32])
484+
comment_size = _unpack_uint16(buffer[32:34])
485+
file_offset = _unpack_uint32(buffer[42:46])
486+
header_size = name_size + extra_size + comment_size
487+
if file_offset > header_offset:
488+
raise ZipImportError(f'bad local header offset: {archive!r}', path=archive)
489+
file_offset += arc_offset
500490

501-
if flags & 0x800:
502-
# UTF-8 file names extension
503-
name = name.decode()
504-
else:
505-
# Historical ZIP filename encoding
506491
try:
507-
name = name.decode('ascii')
508-
except UnicodeDecodeError:
509-
name = name.decode('latin1').translate(cp437_table)
510-
511-
name = name.replace('/', path_sep)
512-
path = _bootstrap_external._path_join(archive, name)
513-
t = (path, compress, data_size, file_size, file_offset, time, date, crc)
514-
files[name] = t
515-
count += 1
492+
name = fp.read(name_size)
493+
except OSError:
494+
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
495+
if len(name) != name_size:
496+
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
497+
# On Windows, calling fseek to skip over the fields we don't use is
498+
# slower than reading the data because fseek flushes stdio's
499+
# internal buffers. See issue #8745.
500+
try:
501+
if len(fp.read(header_size - name_size)) != header_size - name_size:
502+
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
503+
except OSError:
504+
raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive)
505+
506+
if flags & 0x800:
507+
# UTF-8 file names extension
508+
name = name.decode()
509+
else:
510+
# Historical ZIP filename encoding
511+
try:
512+
name = name.decode('ascii')
513+
except UnicodeDecodeError:
514+
name = name.decode('latin1').translate(cp437_table)
515+
516+
name = name.replace('/', path_sep)
517+
path = _bootstrap_external._path_join(archive, name)
518+
t = (path, compress, data_size, file_size, file_offset, time, date, crc)
519+
files[name] = t
520+
count += 1
521+
finally:
522+
fp.seek(start_offset)
516523
_bootstrap._verbose_message('zipimport: found {} names in {!r}', count, archive)
517524
return files
518525

‎Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
On macOS ``python3 /dev/fd/9 9</path/to/script.py`` failed for any script longer than a couple of bytes.

0 commit comments

Comments
 (0)