Skip to content

gh-132775: Add _PyMarshal_GetXIData() #133108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 28, 2025
Merged

gh-132775: Add _PyMarshal_GetXIData() #133108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Include/internal/pycore_crossinterp.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,13 @@ PyAPI_FUNC(_PyBytes_data_t *) _PyBytes_GetXIDataWrapped(
xid_newobjfunc,
_PyXIData_t *);

// _PyObject_GetXIData() for marshal
PyAPI_FUNC(PyObject *) _PyMarshal_ReadObjectFromXIData(_PyXIData_t *);
PyAPI_FUNC(int) _PyMarshal_GetXIData(
PyThreadState *,
PyObject *,
_PyXIData_t *);


/* using cross-interpreter data */

Expand Down
57 changes: 55 additions & 2 deletions Lib/test/_crossinterp_definitions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ def ham_C_closure(z):
ham_C_closure, *_ = eggs_closure_C(2)


FUNCTIONS = [
TOP_FUNCTIONS = [
# shallow
spam_minimal,
spam_full,
Expand All @@ -112,6 +112,8 @@ def ham_C_closure(z):
spam_NC,
spam_CN,
spam_CC,
]
NESTED_FUNCTIONS = [
# inner func
eggs_nested,
eggs_closure,
Expand All @@ -125,6 +127,10 @@ def ham_C_closure(z):
ham_C_nested,
ham_C_closure,
]
FUNCTIONS = [
*TOP_FUNCTIONS,
*NESTED_FUNCTIONS,
]


#######################################
Expand Down Expand Up @@ -157,8 +163,10 @@ async def asyncgen_spam(*args):
gen_spam_1,
gen_spam_2,
async_spam,
coro_spam, # actually FunctionType?
asyncgen_spam,
]
FUNCTION_LIKE_APPLIED = [
coro_spam, # actually FunctionType?
asynccoro_spam, # actually FunctionType?
]

Expand Down Expand Up @@ -202,6 +210,13 @@ def __init__(self, a, b, c):
# __str__
# ...

def __eq__(self, other):
if not isinstance(other, SpamFull):
return NotImplemented
return (self.a == other.a and
self.b == other.b and
self.c == other.c)

@property
def prop(self):
return True
Expand All @@ -222,9 +237,47 @@ class EggsNested:
EggsNested = class_eggs_inner()


TOP_CLASSES = {
Spam: (),
SpamOkay: (),
SpamFull: (1, 2, 3),
SubSpamFull: (1, 2, 3),
SubTuple: ([1, 2, 3],),
}
CLASSES_WITHOUT_EQUALITY = [
Spam,
SpamOkay,
]
BUILTIN_SUBCLASSES = [
SubTuple,
]
NESTED_CLASSES = {
EggsNested: (),
}
CLASSES = {
**TOP_CLASSES,
**NESTED_CLASSES,
}


#######################################
# exceptions

class MimimalError(Exception):
pass


class RichError(Exception):
def __init__(self, msg, value=None):
super().__init__(msg, value)
self.msg = msg
self.value = value

def __eq__(self, other):
if not isinstance(other, RichError):
return NotImplemented
if self.msg != other.msg:
return False
if self.value != other.value:
return False
return True
235 changes: 227 additions & 8 deletions Lib/test/test_crossinterp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@
if isinstance(o, type)]
EXCEPTION_TYPES = [cls for cls in BUILTIN_TYPES
if issubclass(cls, BaseException)]
OTHER_TYPES = [o for n, o in vars(types).items()
if (isinstance(o, type) and
n not in ('DynamicClassAttribute', '_GeneratorWrapper'))]


class _GetXIDataTests(unittest.TestCase):
Expand All @@ -40,16 +43,42 @@ def iter_roundtrip_values(self, values, *, mode=None):
got = _testinternalcapi.restore_crossinterp_data(xid)
yield obj, got

def assert_roundtrip_equal(self, values, *, mode=None):
for obj, got in self.iter_roundtrip_values(values, mode=mode):
self.assertEqual(got, obj)
self.assertIs(type(got), type(obj))

def assert_roundtrip_identical(self, values, *, mode=None):
for obj, got in self.iter_roundtrip_values(values, mode=mode):
# XXX What about between interpreters?
self.assertIs(got, obj)

def assert_roundtrip_equal(self, values, *, mode=None, expecttype=None):
for obj, got in self.iter_roundtrip_values(values, mode=mode):
self.assertEqual(got, obj)
self.assertIs(type(got),
type(obj) if expecttype is None else expecttype)

# def assert_roundtrip_equal_not_identical(self, values, *,
# mode=None, expecttype=None):
# mode = self._resolve_mode(mode)
# for obj in values:
# cls = type(obj)
# with self.subTest(obj):
# got = self._get_roundtrip(obj, mode)
# self.assertIsNot(got, obj)
# self.assertIs(type(got), type(obj))
# self.assertEqual(got, obj)
# self.assertIs(type(got),
# cls if expecttype is None else expecttype)
#
# def assert_roundtrip_not_equal(self, values, *, mode=None, expecttype=None):
# mode = self._resolve_mode(mode)
# for obj in values:
# cls = type(obj)
# with self.subTest(obj):
# got = self._get_roundtrip(obj, mode)
# self.assertIsNot(got, obj)
# self.assertIs(type(got), type(obj))
# self.assertNotEqual(got, obj)
# self.assertIs(type(got),
# cls if expecttype is None else expecttype)

def assert_not_shareable(self, values, exctype=None, *, mode=None):
mode = self._resolve_mode(mode)
for obj in values:
Expand All @@ -66,6 +95,197 @@ def _resolve_mode(self, mode):
return mode


class MarshalTests(_GetXIDataTests):

MODE = 'marshal'

def test_simple_builtin_singletons(self):
self.assert_roundtrip_identical([
True,
False,
None,
Ellipsis,
])
self.assert_not_shareable([
NotImplemented,
])

def test_simple_builtin_objects(self):
self.assert_roundtrip_equal([
# int
*range(-1, 258),
sys.maxsize + 1,
sys.maxsize,
-sys.maxsize - 1,
-sys.maxsize - 2,
2**1000,
# complex
1+2j,
# float
0.0,
1.1,
-1.0,
0.12345678,
-0.12345678,
# bytes
*(i.to_bytes(2, 'little', signed=True)
for i in range(-1, 258)),
b'hello world',
# str
'hello world',
'你好世界',
'',
])
self.assert_not_shareable([
object(),
types.SimpleNamespace(),
])

def test_bytearray(self):
# bytearray is special because it unmarshals to bytes, not bytearray.
self.assert_roundtrip_equal([
bytearray(),
bytearray(b'hello world'),
], expecttype=bytes)

def test_compound_immutable_builtin_objects(self):
self.assert_roundtrip_equal([
# tuple
(),
(1,),
("hello", "world"),
(1, True, "hello"),
# frozenset
frozenset([1, 2, 3]),
])
# nested
self.assert_roundtrip_equal([
# tuple
((1,),),
((1, 2), (3, 4)),
((1, 2), (3, 4), (5, 6)),
# frozenset
frozenset([frozenset([1]), frozenset([2]), frozenset([3])]),
])

def test_compound_mutable_builtin_objects(self):
self.assert_roundtrip_equal([
# list
[],
[1, 2, 3],
# dict
{},
{1: 7, 2: 8, 3: 9},
# set
set(),
{1, 2, 3},
])
# nested
self.assert_roundtrip_equal([
[[1], [2], [3]],
{1: {'a': True}, 2: {'b': False}},
{(1, 2, 3,)},
])

def test_compound_builtin_objects_with_bad_items(self):
bogus = object()
self.assert_not_shareable([
(bogus,),
frozenset([bogus]),
[bogus],
{bogus: True},
{True: bogus},
{bogus},
])

def test_builtin_code(self):
self.assert_roundtrip_equal([
*(f.__code__ for f in defs.FUNCTIONS),
*(f.__code__ for f in defs.FUNCTION_LIKE),
])

def test_builtin_type(self):
shareable = [
StopIteration,
]
types = [
*BUILTIN_TYPES,
*OTHER_TYPES,
]
self.assert_not_shareable(cls for cls in types
if cls not in shareable)
self.assert_roundtrip_identical(cls for cls in types
if cls in shareable)

def test_builtin_function(self):
functions = [
len,
sys.is_finalizing,
sys.exit,
_testinternalcapi.get_crossinterp_data,
]
for func in functions:
assert type(func) is types.BuiltinFunctionType, func

self.assert_not_shareable(functions)

def test_builtin_exception(self):
msg = 'error!'
try:
raise Exception
except Exception as exc:
caught = exc
special = {
BaseExceptionGroup: (msg, [caught]),
ExceptionGroup: (msg, [caught]),
# UnicodeError: (None, msg, None, None, None),
UnicodeEncodeError: ('utf-8', '', 1, 3, msg),
UnicodeDecodeError: ('utf-8', b'', 1, 3, msg),
UnicodeTranslateError: ('', 1, 3, msg),
}
exceptions = []
for cls in EXCEPTION_TYPES:
args = special.get(cls) or (msg,)
exceptions.append(cls(*args))

self.assert_not_shareable(exceptions)
# Note that StopIteration (the type) can be marshalled,
# but its instances cannot.

def test_module(self):
assert type(sys) is types.ModuleType, type(sys)
assert type(defs) is types.ModuleType, type(defs)
assert type(unittest) is types.ModuleType, type(defs)

assert 'emptymod' not in sys.modules
with import_helper.ready_to_import('emptymod', ''):
import emptymod

self.assert_not_shareable([
sys,
defs,
unittest,
emptymod,
])

def test_user_class(self):
self.assert_not_shareable(defs.TOP_CLASSES)

instances = []
for cls, args in defs.TOP_CLASSES.items():
instances.append(cls(*args))
self.assert_not_shareable(instances)

def test_user_function(self):
self.assert_not_shareable(defs.TOP_FUNCTIONS)

def test_user_exception(self):
self.assert_not_shareable([
defs.MimimalError('error!'),
defs.RichError('error!', 42),
])


class ShareableTypeTests(_GetXIDataTests):

MODE = 'xidata'
Expand Down Expand Up @@ -184,6 +404,7 @@ def test_builtin_function(self):

def test_function_like(self):
self.assert_not_shareable(defs.FUNCTION_LIKE)
self.assert_not_shareable(defs.FUNCTION_LIKE_APPLIED)

def test_builtin_wrapper(self):
_wrappers = {
Expand Down Expand Up @@ -243,9 +464,7 @@ def test_class(self):
def test_builtin_type(self):
self.assert_not_shareable([
*BUILTIN_TYPES,
*(o for n, o in vars(types).items()
if (isinstance(o, type) and
n not in ('DynamicClassAttribute', '_GeneratorWrapper'))),
*OTHER_TYPES,
])

def test_exception(self):
Expand Down
Loading
Loading