Skip to content

Commit a57b3d3

Browse files
pablogsalpablogsal
authored
bpo-41625: Expose the splice() system call in the os module (pythonGH-21947)
1 parent cce3f0b commit a57b3d3

File tree

10 files changed

+349
-78
lines changed

10 files changed

+349
-78
lines changed

‎Doc/library/os.rst

+32
Original file line numberDiff line numberDiff line change
@@ -1419,6 +1419,38 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo
14191419
.. versionadded:: 3.3
14201420

14211421

1422+
.. function:: splice(src, dst, count, offset_src=None, offset_dst=None)
1423+
1424+
Transfer *count* bytes from file descriptor *src*, starting from offset
1425+
*offset_src*, to file descriptor *dst*, starting from offset *offset_dst*.
1426+
At least one of the file descriptors must refer to a pipe. If *offset_src*
1427+
is None, then *src* is read from the current position; respectively for
1428+
*offset_dst*. The offset associated to the file descriptor that refers to a
1429+
pipe must be ``None``. The files pointed by *src* and *dst* must reside in
1430+
the same filesystem, otherwise an :exc:`OSError` is raised with
1431+
:attr:`~OSError.errno` set to :data:`errno.EXDEV`.
1432+
1433+
This copy is done without the additional cost of transferring data
1434+
from the kernel to user space and then back into the kernel. Additionally,
1435+
some filesystems could implement extra optimizations. The copy is done as if
1436+
both files are opened as binary.
1437+
1438+
Upon successful completion, returns the number of bytes spliced to or from
1439+
the pipe. A return value of 0 means end of input. If *src* refers to a
1440+
pipe, then this means that there was no data to transfer, and it would not
1441+
make sense to block because there are no writers connected to the write end
1442+
of the pipe.
1443+
1444+
.. availability:: Linux kernel >= 2.6.17 or glibc >= 2.5
1445+
1446+
.. versionadded:: 3.10
1447+
1448+
1449+
.. data:: SPLICE_F_MOVE
1450+
SPLICE_F_NONBLOCK
1451+
SPLICE_F_MORE
1452+
1453+
14221454
.. function:: readv(fd, buffers)
14231455

14241456
Read from a file descriptor *fd* into a number of mutable :term:`bytes-like

‎Doc/whatsnew/3.10.rst

+5
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,11 @@ Added a new function :func:`os.eventfd` and related helpers to wrap the
233233
``eventfd2`` syscall on Linux.
234234
(Contributed by Christian Heimes in :issue:`41001`.)
235235

236+
Added :func:`os.splice()` that allows to move data between two file
237+
descriptors without copying between kernel address space and user
238+
address space, where one of the file descriptors must refer to a
239+
pipe. (Contributed by Pablo Galindo in :issue:`41625`.)
240+
236241
py_compile
237242
----------
238243

‎Lib/test/test_os.py

+117
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,123 @@ def test_copy_file_range_offset(self):
381381
self.assertEqual(read[out_seek:],
382382
data[in_skip:in_skip+i])
383383

384+
@unittest.skipUnless(hasattr(os, 'splice'), 'test needs os.splice()')
385+
def test_splice_invalid_values(self):
386+
with self.assertRaises(ValueError):
387+
os.splice(0, 1, -10)
388+
389+
@unittest.skipUnless(hasattr(os, 'splice'), 'test needs os.splice()')
390+
def test_splice(self):
391+
TESTFN2 = os_helper.TESTFN + ".3"
392+
data = b'0123456789'
393+
394+
create_file(os_helper.TESTFN, data)
395+
self.addCleanup(os_helper.unlink, os_helper.TESTFN)
396+
397+
in_file = open(os_helper.TESTFN, 'rb')
398+
self.addCleanup(in_file.close)
399+
in_fd = in_file.fileno()
400+
401+
read_fd, write_fd = os.pipe()
402+
self.addCleanup(lambda: os.close(read_fd))
403+
self.addCleanup(lambda: os.close(write_fd))
404+
405+
try:
406+
i = os.splice(in_fd, write_fd, 5)
407+
except OSError as e:
408+
# Handle the case in which Python was compiled
409+
# in a system with the syscall but without support
410+
# in the kernel.
411+
if e.errno != errno.ENOSYS:
412+
raise
413+
self.skipTest(e)
414+
else:
415+
# The number of copied bytes can be less than
416+
# the number of bytes originally requested.
417+
self.assertIn(i, range(0, 6));
418+
419+
self.assertEqual(os.read(read_fd, 100), data[:i])
420+
421+
@unittest.skipUnless(hasattr(os, 'splice'), 'test needs os.splice()')
422+
def test_splice_offset_in(self):
423+
TESTFN4 = os_helper.TESTFN + ".4"
424+
data = b'0123456789'
425+
bytes_to_copy = 6
426+
in_skip = 3
427+
428+
create_file(os_helper.TESTFN, data)
429+
self.addCleanup(os_helper.unlink, os_helper.TESTFN)
430+
431+
in_file = open(os_helper.TESTFN, 'rb')
432+
self.addCleanup(in_file.close)
433+
in_fd = in_file.fileno()
434+
435+
read_fd, write_fd = os.pipe()
436+
self.addCleanup(lambda: os.close(read_fd))
437+
self.addCleanup(lambda: os.close(write_fd))
438+
439+
try:
440+
i = os.splice(in_fd, write_fd, bytes_to_copy, offset_src=in_skip)
441+
except OSError as e:
442+
# Handle the case in which Python was compiled
443+
# in a system with the syscall but without support
444+
# in the kernel.
445+
if e.errno != errno.ENOSYS:
446+
raise
447+
self.skipTest(e)
448+
else:
449+
# The number of copied bytes can be less than
450+
# the number of bytes originally requested.
451+
self.assertIn(i, range(0, bytes_to_copy+1));
452+
453+
read = os.read(read_fd, 100)
454+
# 012 are skipped (in_skip)
455+
# 345678 are copied in the file (in_skip + bytes_to_copy)
456+
self.assertEqual(read, data[in_skip:in_skip+i])
457+
458+
@unittest.skipUnless(hasattr(os, 'splice'), 'test needs os.splice()')
459+
def test_splice_offset_out(self):
460+
TESTFN4 = os_helper.TESTFN + ".4"
461+
data = b'0123456789'
462+
bytes_to_copy = 6
463+
out_seek = 3
464+
465+
create_file(os_helper.TESTFN, data)
466+
self.addCleanup(os_helper.unlink, os_helper.TESTFN)
467+
468+
read_fd, write_fd = os.pipe()
469+
self.addCleanup(lambda: os.close(read_fd))
470+
self.addCleanup(lambda: os.close(write_fd))
471+
os.write(write_fd, data)
472+
473+
out_file = open(TESTFN4, 'w+b')
474+
self.addCleanup(os_helper.unlink, TESTFN4)
475+
self.addCleanup(out_file.close)
476+
out_fd = out_file.fileno()
477+
478+
try:
479+
i = os.splice(read_fd, out_fd, bytes_to_copy, offset_dst=out_seek)
480+
except OSError as e:
481+
# Handle the case in which Python was compiled
482+
# in a system with the syscall but without support
483+
# in the kernel.
484+
if e.errno != errno.ENOSYS:
485+
raise
486+
self.skipTest(e)
487+
else:
488+
# The number of copied bytes can be less than
489+
# the number of bytes originally requested.
490+
self.assertIn(i, range(0, bytes_to_copy+1));
491+
492+
with open(TESTFN4, 'rb') as in_file:
493+
read = in_file.read()
494+
# seeked bytes (5) are zero'ed
495+
self.assertEqual(read[:out_seek], b'\x00'*out_seek)
496+
# 012 are skipped (in_skip)
497+
# 345678 are copied in the file (in_skip + bytes_to_copy)
498+
self.assertEqual(read[out_seek:], data[:i])
499+
500+
384501
# Test attributes on return values from os.*stat* family.
385502
class StatAttributeTests(unittest.TestCase):
386503
def setUp(self):

‎Misc/NEWS.d/next/Library/2020-08-24-16-59-04.bpo-41625.Cc967V.rst

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Expose the :c:func:`splice` as :func:`os.splice` in the :mod:`os` module.
2+
Patch by Pablo Galindo

‎Modules/clinic/posixmodule.c.h

+105-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)