From 5e6aa97e7cfe4abd91bf5447972b2e02af00383c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Thu, 18 Oct 2018 02:20:48 +0200 Subject: [PATCH 1/3] [2.7] bpo-24658: Fix read/write greater than 2 GiB on macOS (GH-1705) On macOS, fix reading from and writing into a file with a size larger than 2 GiB. --- Lib/test/test_largefile.py | 11 ++++++- .../2018-10-18-02-20-38.bpo-24658.c7F8Pu.rst | 2 ++ Modules/_io/fileio.c | 29 ++++++++++++++----- 3 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2018-10-18-02-20-38.bpo-24658.c7F8Pu.rst diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 16da8d8f174738..bf9721cf996a21 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -8,7 +8,7 @@ import sys import unittest from test.test_support import run_unittest, TESTFN, verbose, requires, \ - unlink + unlink, bigmemtest import io # C implementation of io import _pyio as pyio # Python implementation of io @@ -48,6 +48,15 @@ def test_seek(self): print('check file size with os.fstat') self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1) + # _pyio.FileIO.readall() uses a temporary bytearry then casted to bytes, + # so memuse=2 is needed + @bigmemtest(minsize=size, memuse=2) + def test_large_read(self, _size): + # bpo-24658: Test that a read greater than 2GB does not fail. + with self.open(TESTFN, "rb") as f: + self.assertEqual(len(f.read()), size + 1) + self.assertEqual(f.tell(), size + 1) + def test_osstat(self): if verbose: print('check file size with os.stat') diff --git a/Misc/NEWS.d/next/macOS/2018-10-18-02-20-38.bpo-24658.c7F8Pu.rst b/Misc/NEWS.d/next/macOS/2018-10-18-02-20-38.bpo-24658.c7F8Pu.rst new file mode 100644 index 00000000000000..a6b50e15e75431 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2018-10-18-02-20-38.bpo-24658.c7F8Pu.rst @@ -0,0 +1,2 @@ +On macOS, fix reading from and writing into a file with a size larger than 2 +GiB. diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 2b40ada195a1f5..1e8fc4d8ddc278 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -37,6 +37,19 @@ #include #endif +#if defined (MS_WINDOWS) || defined(__APPLE__) + /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). + On macOS 10.13, read() and write() with more than INT_MAX bytes + fail with EINVAL (bpo-24658). */ +# define _PY_READ_MAX INT_MAX +# define _PY_WRITE_MAX INT_MAX +#else + /* write() should truncate the input to PY_SSIZE_T_MAX bytes, + but it's safer to do it ourself to have a portable behaviour */ +# define _PY_READ_MAX PY_SSIZE_T_MAX +# define _PY_WRITE_MAX PY_SSIZE_T_MAX +#endif + #if BUFSIZ < (8*1024) #define SMALLCHUNK (8*1024) #elif (BUFSIZ >= (2 << 25)) @@ -604,9 +617,9 @@ fileio_readall(fileio *self) Py_BEGIN_ALLOW_THREADS errno = 0; n = newsize - total; + if (n > _PY_READ_MAX) + n = _PY_READ_MAX; #if defined(MS_WIN64) || defined(MS_WINDOWS) - if (n > INT_MAX) - n = INT_MAX; n = read(self->fd, PyBytes_AS_STRING(result) + total, (int)n); @@ -668,10 +681,10 @@ fileio_read(fileio *self, PyObject *args) return fileio_readall(self); } -#if defined(MS_WIN64) || defined(MS_WINDOWS) - if (size > INT_MAX) - size = INT_MAX; -#endif + if (size > _PY_READ_MAX) { + size = _PY_READ_MAX; + } + bytes = PyBytes_FromStringAndSize(NULL, size); if (bytes == NULL) return NULL; @@ -723,9 +736,9 @@ fileio_write(fileio *self, PyObject *args) Py_BEGIN_ALLOW_THREADS errno = 0; len = pbuf.len; + if (len > _PY_WRITE_MAX) + len = _PY_WRITE_MAX; #if defined(MS_WIN64) || defined(MS_WINDOWS) - if (len > INT_MAX) - len = INT_MAX; n = write(self->fd, pbuf.buf, (int)len); #else n = write(self->fd, pbuf.buf, len); From 9ae079d7ada83348852b629b7eded4ad4665386a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Thu, 18 Oct 2018 09:10:25 +0200 Subject: [PATCH 2/3] patch posixmodule.c and fileio.c, move the #define to pyport.h --- Include/pyport.h | 13 +++++++++++++ Modules/_io/fileio.c | 24 +++++++----------------- Modules/posixmodule.c | 8 ++++++-- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index 0c78a1e5b668d3..6987a577678a2c 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -947,4 +947,17 @@ typedef struct fd_set { #define Py_ULL(x) Py_LL(x##U) #endif +#if defined (MS_WINDOWS) || defined(__APPLE__) + /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). + On macOS 10.13, read() and write() with more than INT_MAX bytes + fail with EINVAL (bpo-24658). */ +# define _PY_READ_MAX INT_MAX +# define _PY_WRITE_MAX INT_MAX +#else + /* write() should truncate the input to PY_SSIZE_T_MAX bytes, + but it's safer to do it ourself to have a portable behaviour */ +# define _PY_READ_MAX PY_SSIZE_T_MAX +# define _PY_WRITE_MAX PY_SSIZE_T_MAX +#endif + #endif /* Py_PYPORT_H */ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 1e8fc4d8ddc278..eb117bc5aa2d73 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -37,19 +37,6 @@ #include #endif -#if defined (MS_WINDOWS) || defined(__APPLE__) - /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). - On macOS 10.13, read() and write() with more than INT_MAX bytes - fail with EINVAL (bpo-24658). */ -# define _PY_READ_MAX INT_MAX -# define _PY_WRITE_MAX INT_MAX -#else - /* write() should truncate the input to PY_SSIZE_T_MAX bytes, - but it's safer to do it ourself to have a portable behaviour */ -# define _PY_READ_MAX PY_SSIZE_T_MAX -# define _PY_WRITE_MAX PY_SSIZE_T_MAX -#endif - #if BUFSIZ < (8*1024) #define SMALLCHUNK (8*1024) #elif (BUFSIZ >= (2 << 25)) @@ -528,9 +515,10 @@ fileio_readinto(fileio *self, PyObject *args) len = pbuf.len; Py_BEGIN_ALLOW_THREADS errno = 0; + if (len > _PY_READ_MAX) { + len = _PY_READ_MAX; + } #if defined(MS_WIN64) || defined(MS_WINDOWS) - if (len > INT_MAX) - len = INT_MAX; n = read(self->fd, pbuf.buf, (int)len); #else n = read(self->fd, pbuf.buf, len); @@ -617,8 +605,9 @@ fileio_readall(fileio *self) Py_BEGIN_ALLOW_THREADS errno = 0; n = newsize - total; - if (n > _PY_READ_MAX) + if (n > _PY_READ_MAX) { n = _PY_READ_MAX; + } #if defined(MS_WIN64) || defined(MS_WINDOWS) n = read(self->fd, PyBytes_AS_STRING(result) + total, @@ -736,8 +725,9 @@ fileio_write(fileio *self, PyObject *args) Py_BEGIN_ALLOW_THREADS errno = 0; len = pbuf.len; - if (len > _PY_WRITE_MAX) + if (len > _PY_WRITE_MAX) { len = _PY_WRITE_MAX; + } #if defined(MS_WIN64) || defined(MS_WINDOWS) n = write(self->fd, pbuf.buf, (int)len); #else diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7a1a6945c102eb..eebf9c9c5b539e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6805,6 +6805,9 @@ posix_read(PyObject *self, PyObject *args) return posix_error(); } Py_BEGIN_ALLOW_THREADS + if (size > _PY_READ_MAX) { + size = _PY_READ_MAX; + } n = read(fd, PyString_AsString(buffer), size); Py_END_ALLOW_THREADS if (n < 0) { @@ -6836,9 +6839,10 @@ posix_write(PyObject *self, PyObject *args) } len = pbuf.len; Py_BEGIN_ALLOW_THREADS + if (len > _PY_WRITE_MAX) { + len = _PY_WRITE_MAX; + } #if defined(MS_WIN64) || defined(MS_WINDOWS) - if (len > INT_MAX) - len = INT_MAX; size = write(fd, pbuf.buf, (int)len); #else size = write(fd, pbuf.buf, len); From c6a483e08ecc63d3e370ac0f688a2ab5c45a8dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Thu, 18 Oct 2018 10:03:22 +0200 Subject: [PATCH 3/3] add test_large_read in the tests discovery --- Lib/test/test_largefile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index bf9721cf996a21..57725f2fc71999 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -195,6 +195,7 @@ class TestCase(LargeFileTest): suite.addTest(TestCase('test_osstat')) suite.addTest(TestCase('test_seek_read')) suite.addTest(TestCase('test_lseek')) + suite.addTest(TestCase('test_large_read')) with _open(TESTFN, 'wb') as f: if hasattr(f, 'truncate'): suite.addTest(TestCase('test_truncate'))