Skip to content

Commit fdee7b7

Browse files
colesburycolesbury
authored
gh-112532: Require mimalloc in --disable-gil builds (gh-112883)
1 parent fed294c commit fdee7b7

File tree

10 files changed

+76
-12
lines changed

10 files changed

+76
-12
lines changed

‎Include/internal/pycore_pymem_init.h

+13-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,19 @@ extern void * _PyMem_RawRealloc(void *, void *, size_t);
1818
extern void _PyMem_RawFree(void *, void *);
1919
#define PYRAW_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree}
2020

21-
#if defined(WITH_PYMALLOC)
21+
#ifdef Py_GIL_DISABLED
22+
// Py_GIL_DISABLED requires mimalloc
23+
extern void* _PyObject_MiMalloc(void *, size_t);
24+
extern void* _PyObject_MiCalloc(void *, size_t, size_t);
25+
extern void _PyObject_MiFree(void *, void *);
26+
extern void* _PyObject_MiRealloc(void *, void *, size_t);
27+
# define PYOBJ_ALLOC {NULL, _PyObject_MiMalloc, _PyObject_MiCalloc, _PyObject_MiRealloc, _PyObject_MiFree}
28+
extern void* _PyMem_MiMalloc(void *, size_t);
29+
extern void* _PyMem_MiCalloc(void *, size_t, size_t);
30+
extern void _PyMem_MiFree(void *, void *);
31+
extern void* _PyMem_MiRealloc(void *, void *, size_t);
32+
# define PYMEM_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree}
33+
#elif defined(WITH_PYMALLOC)
2234
extern void* _PyObject_Malloc(void *, size_t);
2335
extern void* _PyObject_Calloc(void *, size_t, size_t);
2436
extern void _PyObject_Free(void *, void *);

‎Lib/test/support/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1844,7 +1844,7 @@ def restore(self):
18441844

18451845
def with_pymalloc():
18461846
import _testcapi
1847-
return _testcapi.WITH_PYMALLOC
1847+
return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED
18481848

18491849

18501850
def with_mimalloc():

‎Lib/test/test_capi/test_mem.py

+7
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,8 @@ class C(): pass
152152
self.assertGreaterEqual(count, i*5-2)
153153

154154

155+
# Py_GIL_DISABLED requires mimalloc (not malloc)
156+
@unittest.skipIf(support.Py_GIL_DISABLED, 'need malloc')
155157
class PyMemMallocDebugTests(PyMemDebugTests):
156158
PYTHONMALLOC = 'malloc_debug'
157159

@@ -161,6 +163,11 @@ class PyMemPymallocDebugTests(PyMemDebugTests):
161163
PYTHONMALLOC = 'pymalloc_debug'
162164

163165

166+
@unittest.skipUnless(support.with_mimalloc(), 'need mimaloc')
167+
class PyMemMimallocDebugTests(PyMemDebugTests):
168+
PYTHONMALLOC = 'mimalloc_debug'
169+
170+
164171
@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
165172
class PyMemDefaultTests(PyMemDebugTests):
166173
# test default allocator of Python compiled in debug mode

‎Lib/test/test_cmd_line.py

+12-3
Original file line numberDiff line numberDiff line change
@@ -738,6 +738,8 @@ def test_xdev(self):
738738
out = self.run_xdev("-c", code, check_exitcode=False)
739739
if support.with_pymalloc():
740740
alloc_name = "pymalloc_debug"
741+
elif support.Py_GIL_DISABLED:
742+
alloc_name = "mimalloc_debug"
741743
else:
742744
alloc_name = "malloc_debug"
743745
self.assertEqual(out, alloc_name)
@@ -814,9 +816,13 @@ def check_pythonmalloc(self, env_var, name):
814816
@support.cpython_only
815817
def test_pythonmalloc(self):
816818
# Test the PYTHONMALLOC environment variable
819+
malloc = not support.Py_GIL_DISABLED
817820
pymalloc = support.with_pymalloc()
818821
mimalloc = support.with_mimalloc()
819-
if pymalloc:
822+
if support.Py_GIL_DISABLED:
823+
default_name = 'mimalloc_debug' if support.Py_DEBUG else 'mimalloc'
824+
default_name_debug = 'mimalloc_debug'
825+
elif pymalloc:
820826
default_name = 'pymalloc_debug' if support.Py_DEBUG else 'pymalloc'
821827
default_name_debug = 'pymalloc_debug'
822828
else:
@@ -826,9 +832,12 @@ def test_pythonmalloc(self):
826832
tests = [
827833
(None, default_name),
828834
('debug', default_name_debug),
829-
('malloc', 'malloc'),
830-
('malloc_debug', 'malloc_debug'),
831835
]
836+
if malloc:
837+
tests.extend([
838+
('malloc', 'malloc'),
839+
('malloc_debug', 'malloc_debug'),
840+
])
832841
if pymalloc:
833842
tests.extend((
834843
('pymalloc', 'pymalloc'),

‎Lib/test/test_embed.py

+10-4
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@
2323
PYMEM_ALLOCATOR_NOT_SET = 0
2424
PYMEM_ALLOCATOR_DEBUG = 2
2525
PYMEM_ALLOCATOR_MALLOC = 3
26+
PYMEM_ALLOCATOR_MIMALLOC = 7
27+
if support.Py_GIL_DISABLED:
28+
ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MIMALLOC
29+
else:
30+
ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MALLOC
31+
2632
Py_STATS = hasattr(sys, '_stats_on')
2733

2834
# _PyCoreConfig_InitCompatConfig()
@@ -841,7 +847,7 @@ def test_init_global_config(self):
841847

842848
def test_init_from_config(self):
843849
preconfig = {
844-
'allocator': PYMEM_ALLOCATOR_MALLOC,
850+
'allocator': ALLOCATOR_FOR_CONFIG,
845851
'utf8_mode': 1,
846852
}
847853
config = {
@@ -908,7 +914,7 @@ def test_init_from_config(self):
908914

909915
def test_init_compat_env(self):
910916
preconfig = {
911-
'allocator': PYMEM_ALLOCATOR_MALLOC,
917+
'allocator': ALLOCATOR_FOR_CONFIG,
912918
}
913919
config = {
914920
'use_hash_seed': 1,
@@ -942,7 +948,7 @@ def test_init_compat_env(self):
942948

943949
def test_init_python_env(self):
944950
preconfig = {
945-
'allocator': PYMEM_ALLOCATOR_MALLOC,
951+
'allocator': ALLOCATOR_FOR_CONFIG,
946952
'utf8_mode': 1,
947953
}
948954
config = {
@@ -984,7 +990,7 @@ def test_init_env_dev_mode(self):
984990
api=API_COMPAT)
985991

986992
def test_init_env_dev_mode_alloc(self):
987-
preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC)
993+
preconfig = dict(allocator=ALLOCATOR_FOR_CONFIG)
988994
config = dict(dev_mode=1,
989995
faulthandler=1,
990996
warnoptions=['default'])

‎Lib/test/test_os.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -5080,7 +5080,10 @@ def test_fork(self):
50805080
support.wait_process(pid, exitcode=0)
50815081
"""
50825082
assert_python_ok("-c", code)
5083-
assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug")
5083+
if support.Py_GIL_DISABLED:
5084+
assert_python_ok("-c", code, PYTHONMALLOC="mimalloc_debug")
5085+
else:
5086+
assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug")
50845087

50855088
@unittest.skipUnless(sys.platform in ("linux", "darwin"),
50865089
"Only Linux and macOS detect this today.")

‎Objects/obmalloc.c

+13-2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616
# include "mimalloc/internal.h" // for stats
1717
#endif
1818

19+
#if defined(Py_GIL_DISABLED) && !defined(WITH_MIMALLOC)
20+
# error "Py_GIL_DISABLED requires WITH_MIMALLOC"
21+
#endif
22+
1923
#undef uint
2024
#define uint pymem_uint
2125

@@ -153,7 +157,12 @@ void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
153157
# define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free}
154158
#endif // WITH_PYMALLOC
155159

156-
#if defined(WITH_PYMALLOC)
160+
#if defined(Py_GIL_DISABLED)
161+
// Py_GIL_DISABLED requires using mimalloc for "mem" and "obj" domains.
162+
# define PYRAW_ALLOC MALLOC_ALLOC
163+
# define PYMEM_ALLOC MIMALLOC_ALLOC
164+
# define PYOBJ_ALLOC MIMALLOC_OBJALLOC
165+
#elif defined(WITH_PYMALLOC)
157166
# define PYRAW_ALLOC MALLOC_ALLOC
158167
# define PYMEM_ALLOC PYMALLOC_ALLOC
159168
# define PYOBJ_ALLOC PYMALLOC_ALLOC
@@ -350,7 +359,7 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)
350359
else if (strcmp(name, "debug") == 0) {
351360
*allocator = PYMEM_ALLOCATOR_DEBUG;
352361
}
353-
#ifdef WITH_PYMALLOC
362+
#if defined(WITH_PYMALLOC) && !defined(Py_GIL_DISABLED)
354363
else if (strcmp(name, "pymalloc") == 0) {
355364
*allocator = PYMEM_ALLOCATOR_PYMALLOC;
356365
}
@@ -366,12 +375,14 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)
366375
*allocator = PYMEM_ALLOCATOR_MIMALLOC_DEBUG;
367376
}
368377
#endif
378+
#ifndef Py_GIL_DISABLED
369379
else if (strcmp(name, "malloc") == 0) {
370380
*allocator = PYMEM_ALLOCATOR_MALLOC;
371381
}
372382
else if (strcmp(name, "malloc_debug") == 0) {
373383
*allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;
374384
}
385+
#endif
375386
else {
376387
/* unknown allocator */
377388
return -1;

‎Programs/_testembed.c

+12
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,11 @@ static int test_init_from_config(void)
576576
_PyPreConfig_InitCompatConfig(&preconfig);
577577

578578
putenv("PYTHONMALLOC=malloc_debug");
579+
#ifndef Py_GIL_DISABLED
579580
preconfig.allocator = PYMEM_ALLOCATOR_MALLOC;
581+
#else
582+
preconfig.allocator = PYMEM_ALLOCATOR_MIMALLOC;
583+
#endif
580584

581585
putenv("PYTHONUTF8=0");
582586
Py_UTF8Mode = 0;
@@ -765,7 +769,11 @@ static int test_init_dont_parse_argv(void)
765769
static void set_most_env_vars(void)
766770
{
767771
putenv("PYTHONHASHSEED=42");
772+
#ifndef Py_GIL_DISABLED
768773
putenv("PYTHONMALLOC=malloc");
774+
#else
775+
putenv("PYTHONMALLOC=mimalloc");
776+
#endif
769777
putenv("PYTHONTRACEMALLOC=2");
770778
putenv("PYTHONPROFILEIMPORTTIME=1");
771779
putenv("PYTHONNODEBUGRANGES=1");
@@ -851,7 +859,11 @@ static int test_init_env_dev_mode_alloc(void)
851859
/* Test initialization from environment variables */
852860
Py_IgnoreEnvironmentFlag = 0;
853861
set_all_env_vars_dev_mode();
862+
#ifndef Py_GIL_DISABLED
854863
putenv("PYTHONMALLOC=malloc");
864+
#else
865+
putenv("PYTHONMALLOC=mimalloc");
866+
#endif
855867
_testembed_Py_InitializeFromConfig();
856868
dump_config();
857869
Py_Finalize();

‎configure

+2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎configure.ac

+2
Original file line numberDiff line numberDiff line change
@@ -4558,6 +4558,8 @@ if test "$with_mimalloc" != no; then
45584558
with_mimalloc=yes
45594559
AC_DEFINE([WITH_MIMALLOC], [1], [Define if you want to compile in mimalloc memory allocator.])
45604560
AC_SUBST([MIMALLOC_HEADERS], ['$(MIMALLOC_HEADERS)'])
4561+
elif test "$disable_gil" = "yes"; then
4562+
AC_MSG_ERROR([--disable-gil requires mimalloc memory allocator (--with-mimalloc).])
45614563
fi
45624564

45634565
AC_MSG_RESULT([$with_mimalloc])

0 commit comments

Comments
 (0)