From b33d54f2ab2b67debb75b993efd821aeaf029ab8 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Tue, 25 Apr 2023 15:26:20 -0600 Subject: [PATCH 1/2] Add a benchmark for runtime-checkable protocols --- pyperformance/data-files/benchmarks/MANIFEST | 1 + .../pyproject.toml | 9 + .../run_benchmark.py | 194 ++++++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml create mode 100644 pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py diff --git a/pyperformance/data-files/benchmarks/MANIFEST b/pyperformance/data-files/benchmarks/MANIFEST index d472c2c1..01c87603 100644 --- a/pyperformance/data-files/benchmarks/MANIFEST +++ b/pyperformance/data-files/benchmarks/MANIFEST @@ -69,6 +69,7 @@ sympy telco tomli_loads tornado_http +typing_runtime_protocols unpack_sequence unpickle unpickle_list diff --git a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml new file mode 100644 index 00000000..a69cf598 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "pyperformance_bm_typing_runtime_protocols" +requires-python = ">=3.8" +dependencies = ["pyperf"] +urls = {repository = "https://github.com/python/pyperformance"} +dynamic = ["version"] + +[tool.pyperformance] +name = "typing_runtime_protocols" diff --git a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py new file mode 100644 index 00000000..b7c8412a --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py @@ -0,0 +1,194 @@ +""" +Test the performance of isinstance() checks against runtime-checkable protocols. + +For programmes that make extensive use of this feature, +these calls can easily become a bottleneck. +See https://github.com/python/cpython/issues/74690 + +The following situations all exercise different code paths +in typing._ProtocolMeta.__instancecheck__, +so each is tested in this benchmark: + + (1) Comparing an instance of a class that directly inherits + from a protocol to that protocol. + (2) Comparing an instance of a class that fulfils the interface + of a protocol using instance attributes. + (3) Comparing an instance of a class that fulfils the interface + of a protocol using class attributes. + (4) Comparing an instance of a class that fulfils the interface + of a protocol using properties. + +Protocols with callable and non-callable members also +exercise different code paths in _ProtocolMeta.__instancecheck__, +so are also tested separately. +""" + +import time +from typing import Protocol, runtime_checkable + +import pyperf + + +################################################## +# Protocols to call isinstance() against +################################################## + + +@runtime_checkable +class HasX(Protocol): + """A runtime-checkable protocol with a single non-callable member""" + x: int + +@runtime_checkable +class HasManyAttributes(Protocol): + """A runtime-checkable protocol with many non-callable members""" + a: int + b: int + c: int + d: int + e: int + +@runtime_checkable +class SupportsInt(Protocol): + """A runtime-checkable protocol with a single callable member""" + def __int__(self) -> int: ... + +@runtime_checkable +class SupportsManyMethods(Protocol): + """A runtime-checkable protocol with many callable members""" + def one(self) -> int: ... + def two(self) -> str: ... + def three(self) -> bytes: ... + def four(self) -> memoryview: ... + def five(self) -> bytearray: ... + +@runtime_checkable +class SupportsIntAndX(Protocol): + """A runtime-checkable protocol with a mix of callable and non-callable members""" + x: int + def __int__(self) -> int: ... + + +################################################## +# Classes for comparing against the protocols +################################################## + + +class Empty: + """Empty class with no attributes""" + +class PropertyX: + """Class with a property x""" + @property + def x(self) -> int: return 42 + +class HasIntMethod: + """Class with an __int__ method""" + def __int__(self): return 42 + +class HasManyMethods: + """Class with many methods""" + def one(self): return 42 + def two(self): return "42" + def three(self): return b"42" + def four(self): return memoryview(b"42") + def five(self): return bytearray(b"42") + +class PropertyXWithInt: + """Class with a property x and an __int__ method""" + @property + def x(self) -> int: return 42 + def __int__(self): return 42 + +class ClassVarX: + """Class with a ClassVar x""" + x = 42 + +class ClassVarXWithInt: + """Class with a ClassVar x and an __int__ method""" + x = 42 + def __int__(self): return 42 + +class InstanceVarX: + """Class with an instance var x""" + def __init__(self): + self.x = 42 + +class ManyInstanceVars: + """Class with many instance vars""" + def __init__(self): + for attr in 'abcde': + setattr(self, attr, 42) + +class InstanceVarXWithInt: + """Class with an instance var x and an __int__ method""" + def __init__(self): + self.x = 42 + def __int__(self): + return 42 + +class NominalX(HasX): + """Class that explicitly subclasses HasX""" + def __init__(self): + self.x = 42 + +class NominalSupportsInt(SupportsInt): + """Class that explicitly subclasses SupportsInt""" + def __int__(self): + return 42 + +class NominalXWithInt(SupportsIntAndX): + """Class that explicitly subclasses NominalXWithInt""" + def __init__(self): + self.x = 42 + + +################################################## +# Time to test the performance of isinstance()! +################################################## + + +def bench_protocols(loops: int) -> float: + instances = { + obj: obj() + for obj in globals().values() + if isinstance(obj, type) and not getattr(obj, "_is_protocol", False) + } + + t0 = time.perf_counter() + + for cls in Empty, PropertyX, InstanceVarX, NominalX: + instance = instances[cls] + for _ in range(loops): + isinstance(instance, HasX) + + for cls in Empty, ManyInstanceVars: + instance = instances[cls] + for _ in range(loops): + isinstance(instance, HasManyAttributes) + + for cls in Empty, HasIntMethod, NominalSupportsInt: + instance = instances[cls] + for _ in range(loops): + isinstance(instance, SupportsInt) + + for cls in Empty, HasManyMethods: + instance = instances[cls] + for _ in range(loops): + isinstance(instance, SupportsManyMethods) + + for cls in Empty, PropertyXWithInt, InstanceVarXWithInt, NominalXWithInt: + instance = instances[cls] + for _ in range(loops): + isinstance(instance, SupportsIntAndX) + + return time.perf_counter() - t0 + + +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata["description"] = ( + "Test the performance of isinstance() checks " + "against runtime-checkable protocols" + ) + runner.bench_time_func("typing_runtime_protocols", bench_protocols) From 04f7bbb9b8085fe8909cb085416e814474fec69f Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Tue, 25 Apr 2023 16:27:24 -0600 Subject: [PATCH 2/2] Address review --- .../run_benchmark.py | 44 +++++++------------ 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py index b7c8412a..dd8e7ef3 100644 --- a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py +++ b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py @@ -149,38 +149,24 @@ def __init__(self): def bench_protocols(loops: int) -> float: - instances = { - obj: obj() - for obj in globals().values() - if isinstance(obj, type) and not getattr(obj, "_is_protocol", False) - } + protocols = [ + HasX, HasManyAttributes, SupportsInt, SupportsManyMethods, SupportsIntAndX + ] + instances = [ + cls() + for cls in ( + Empty, PropertyX, HasIntMethod, HasManyMethods, PropertyXWithInt, + ClassVarX, ClassVarXWithInt, InstanceVarX, ManyInstanceVars, + InstanceVarXWithInt, NominalX, NominalSupportsInt, NominalXWithInt + ) + ] t0 = time.perf_counter() - for cls in Empty, PropertyX, InstanceVarX, NominalX: - instance = instances[cls] - for _ in range(loops): - isinstance(instance, HasX) - - for cls in Empty, ManyInstanceVars: - instance = instances[cls] - for _ in range(loops): - isinstance(instance, HasManyAttributes) - - for cls in Empty, HasIntMethod, NominalSupportsInt: - instance = instances[cls] - for _ in range(loops): - isinstance(instance, SupportsInt) - - for cls in Empty, HasManyMethods: - instance = instances[cls] - for _ in range(loops): - isinstance(instance, SupportsManyMethods) - - for cls in Empty, PropertyXWithInt, InstanceVarXWithInt, NominalXWithInt: - instance = instances[cls] - for _ in range(loops): - isinstance(instance, SupportsIntAndX) + for _ in range(loops): + for protocol in protocols: + for instance in instances: + isinstance(instance, protocol) return time.perf_counter() - t0