From 79a38e05325b0fedde11108d2b475840683a712d Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Sat, 10 Feb 2024 19:02:08 -0800 Subject: [PATCH 01/11] Added draft chapter to typing spec for constructors. --- docs/spec/constructors.rst | 375 +++++++++++++++++++++++++++++++++++++ docs/spec/index.rst | 1 + 2 files changed, 376 insertions(+) create mode 100644 docs/spec/constructors.rst diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst new file mode 100644 index 00000000..925692eb --- /dev/null +++ b/docs/spec/constructors.rst @@ -0,0 +1,375 @@ +Constructors +============ + +Calls to constructors require special handling within type checkers. + +Constructor Calls +----------------- + +At runtime, a call to a class' constructor typically results in the invocation of +three methods in the following order: +1. The ``__call__`` method of the metaclass (which is typically supplied by the + ``type`` class but can be overridden by a custom metaclass) +2. The ``__new__`` static method of the class +3. The ``__init__`` instance method of the class + +Type checkers should mirror this runtime behavior when analyzing a constructor +call. + +Metaclass ``__call__`` Method +============================= + +When evaluating a constructor call, a type checker should first check if the +class has a custom metaclass (a subclass of ``type``) that defines a ``__call__`` +method. If so, it should evaluate the call of this method using the supplied +arguments. If the metaclass is ``type``, this step can be skipped. + +If the evaluated return type of the ``__call__`` method is something other than +``Any`` or an instance of the class being constructed, a type checker should +assume that the metaclass ``__call__`` method is overriding ``type.__call__`` +in some special manner, and it should not attempt to evaluate the ``__new__`` +or ``__init__`` methods on the class. For example, some metaclass ``__call__`` +methods are annotated to return ``NoReturn`` to indicate that constructor +calls are not supported for that class. + + :: + + class Meta(type): + def __call__(cls, *args, **kwargs) -> NoReturn: + raise TypeError("Cannot instantiate class") + + class MyClass(metaclass=Meta): + def __new__(cls, *args, **kwargs) -> Self: + return super().__new__(cls, *args, **kwargs) + + assert_type(MyClass(), Never) + + +``__new__`` Method +================== + +After the metaclass ``__call__`` method has been evaluated, a type checker +should evaluate the ``__new__`` method of the class using the supplied arguments. +If the class is generic and explicitly specialized, the type checker +should partially specialize the ``__new__`` method using the supplied type +arguments. If the class is not explicitly specialized, class-scoped type +variables should be solved using the supplied arguments passed to the constructor +call. + + :: + + class MyClass[T]: + def __new__(cls, x: T) -> Self: + return super().__new__(cls) + + # Constructor calls for specialized classes + assert_type(MyClass[int](1), MyClass[int]) + assert_type(MyClass[float](1), MyClass[float]) + MyClass[int](1.0) # Type error + + # Constructor calls for non-specialized classes + assert_type(MyClass(1), MyClass[int]) + assert_type(MyClass(1.0), MyClass[float]) + +If any class-scoped type variables are not solved when evaluating the ``__new__`` +method call using the supplied arguments, these type variables should be left +unsolved, allowing the ``__init__`` method to be used to solve them. + + :: + + class MyClass[T]: + def __new__(cls, *args, **kwargs) -> Self: + return super().__new__(cls) + + def __init__(self, x: T) -> None: + pass + + assert_type(MyClass(1), MyClass[int]) + assert_type(MyClass(""), MyClass[str]) + +For most classes, the return type for ``__new__`` method is typically ``Self``, +but other types are also allowed. For example, the ``__new__`` method may return +an instance of a subclass or an instance of some completely unrelated class. + +If the return type of the ``__new__`` method is specified as ``Any`` (or the +return type is unspecified and not inferred), a type checker should +assume that the return type is ``Self``, and it should proceed to evaluate the +``__init__`` method. + + :: + + class MyClass: + def __new__(cls, *args, **kwargs) -> Any: + return super().__new__(*args, **kwargs) + + def __init__(self): + pass + + assert_type(MyClass(), MyClass) + +If the ``__new__`` method returns a type that is not an instance of the class +being constructed (or a subclass thereof), a type checker should assume that +the ``__init__`` method will not be called. This is consistent with the runtime +behavior of the ``type.__call__`` method. + + :: + + class MyClass: + def __new__(cls) -> int: + return 0 + + # The __init__ method will not be called in this case, so + # it should not be evaluated. + def __init__(self, x: int): + pass + + assert_type(MyClass(), int) + +If the class is generic, it is possible for a ``__new__`` method to override +the specialized class type and return a class instance that is specialized +with different type arguments. + + :: + + class MyClass[T]: + def __new__(cls, *args, **kwargs) -> "MyClass[list[T]]": + ... + + assert_type(MyClass[int](), MyClass[list[int]]) + +If the ``cls`` parameter within the ``__new__`` method is not annotated, type +checkers should infer a type of ``type[Self]``. Regardless of whether the +type of the ``cls`` parameter is explicit or inferred, the type checker should +bind the class being constructed to this parameter and report any type errors +that arise during binding. + + :: + + class MyClass[T]: + def __new__(cls: "type[MyClass[int]]") -> "MyClass[int]": ... + + MyClass() # OK + MyClass[int]() # OK + MyClass[str]() # Type Error + + +``__init__`` Method +=================== + +After evaluating the ``__new__`` method, a type checker should evaluate the +``__init__`` method using the supplied arguments. If the class is +generic and explicitly specialized (or specialized via the ``__new__`` method +return type), the type checker should partially specialize the ``__init__`` +method using the supplied type arguments. If the class is not explicitly +specialized, class-scoped type variables should be solved using the supplied +arguments passed to the constructor call. + + :: + + class MyClass[T]: + def __init__(self, x: T) -> None: + ... + + # Constructor calls for specialized classes + assert_type(MyClass[int](1), MyClass[int]) + assert_type(MyClass[float](1), MyClass[float]) + MyClass[int](1.0) # Type error + + # Constructor calls for non-specialized classes + assert_type(MyClass(1), MyClass[int]) + assert_type(MyClass(1.0), MyClass[float]) + +If the ``self`` parameter within the ``__init__`` method is not annotated, type +checkers should infer a type of ``Self``. Regardless of whether the ``self`` +parameter type is explicit or inferred, a type checker should bind the class +being constructed to this parameter and report any type errors that arise +during binding. + + :: + + class MyClass[T]: + def __init__(self: "MyClass[int]") -> "None": ... + + MyClass() # OK + MyClass[int]() # OK + MyClass[str]() # Type Error + +The return type for ``__init__`` is always ``None``, which means the +method cannot influence the return type of the constructor call by specifying +a return type. To work around this limitation, type checkers should allow +the ``self`` parameter to be annotated with a type that influences the resulting +type of the constructor call. This can be used in overloads to influence the +constructor return type for each overload. + + :: + + class MyClass1[T]: + @overload + def __init__(self: "MyClass1[list[int]]", value: int) -> None: ... + @overload + def __init__(self: "MyClass1[set[str]]", value: str) -> None: ... + @overload + def __init__(self, value: T) -> None: ... + + + assert_type(MyClass1(0), MyClass1[list[int]]) + assert_type(MyClass1[int](3), MyClass1[int]) + assert_type(MyClass1(""), MyClass1[set[str]]) + assert_type(MyClass1(3.0), MyClass1[float]) + + +Function-scoped type variables can also be used in the ``self`` +annotation of an ``__init__`` method to influence the return type of the +constructor call. + + :: + + class MyClass2[T1, T2]: + def __init__[V1, V2](self: "MyClass2[V1, V2]", value1: V1, value2: V2) -> None: ... + + assert_type(MyClass2(0, ""), MyClass2[int, str]) + assert_type(MyClass2[int, str](0, ""), MyClass2[int, str]) + + class MyClass3[T1, T2]: + def __init__[V1, V2](self: "MyClass3[V2, V1]", value1: V1, value2: V2) -> None: ... + + assert_type(MyClass3(0, ""), MyClass3[str, int]) + assert_type(MyClass3[str, int](0, ""), MyClass3[str, int]) + + +Class-scoped type variables should not be used in the ``self`` annotation +because such use can lead to ambiguous or nonsensical type evaluation results. +Type checkers should report an error if a class-scoped type variable is used +within a type annotation for the ``self`` parameter in an ``__init__`` method. + + :: + + class MyClass4[T1, T2]: + # The ``self`` annotation should result in a type error + def __init__(self: "MyClass4[T2, T1]") -> None: ... + + +Constructor Calls for type[T] +----------------------------- + +When a value of type ``type[T]`` (where ``T`` is a type variable or a concrete +class) is called, a type checker should evaluate the constructor call as if +it is being made on the class ``T`` (or the class the represents the upper bound +of type variable ``T``). This means the type checker should use ``__call__`` +method of ``T``'s metaclass and the ``__new__`` and ``__init__`` methods of ``T`` +to evaluate the constructor call. + +It should be noted that such code could be unsafe because The type ``type[T]`` +may represent subclasses of ``T``, and those subclasses could redefine the +``__new__`` and ``__init__`` methods in a way that is incompatible with the +base class. Likewise, the metaclass of ``T`` could redefine the ``__call__`` +method in a way that is incompatible with the base metaclass. + + +Specialization During Construction +---------------------------------- + +As discussed above, if a class is generic and not explicitly specialized, its +type variables should be solved using the arguments passed to the ``__new__`` +and ``__init__`` methods. If one or more type variables are not solved during +these method evaluations, they should take on their default values. + + :: + + T1 = TypeVar("T1") + T2 = TypeVar("T2") + T3 = TypeVar("T3", default=str) + + class MyClass1(Generic[T1, T2]): + def __new__(cls, x: T1) -> Self: ... + + assert_type(MyClass1(1), MyClass1[int, Any]) + + class MyClass2(Generic[T1, T3]): + def __new__(cls, x: T1) -> Self: ... + + assert_type(MyClass2(1), MyClass2[int, str]) + + +Converting a Constructor to Callable +------------------------------------ + +Class objects are callable, which means they are compatible with callable types. + + :: + + def accepts_callable[**P, R](cb: Callable[P, R]) -> Callable[P, R]: + return cb + + class MyClass: + def __init__(self, x: int) -> None: + pass + + reveal_type(accepts_callable(MyClass)) # ``def (x: int) -> MyClass`` + +When converting a class to a callable type, a type checker should use the +following rules: + +1. If the class defines an ``__init__`` method or inherits an ``__init__`` + method from a base class other than ``object``, the callable type should be + synthesized from the parameters of the ``__init__`` method after it is bound + to the class instance. The return type of the synthesized callable should be + the class itself. + +2. If the class does not define or inherit an ``__init__`` method from a base + class other than the ``object`` class, the callable type should be synthesized + from the parameters of the ``__new__`` method after it is bound to the class. + The return type of the synthesized callable should come from the bound + ``__new__`` method. + + :: + + class A: + pass + + class B: + def __new__(cls, *args, **kwargs) -> Self: + ... + + def __init__(self, x: int) -> None: + ... + + class C: + def __new__(cls, x: int) -> int: + ... + + reveal_type(accepts_callable(A)) # ``def () -> A`` + reveal_type(accepts_callable(B)) # ``def (x: int) -> B`` + reveal_type(accepts_callable(C)) # ``def (x: int) -> int`` + + +If the ``__init__`` or ``__new__`` method is overloaded, the callable +type should be synthesized from the overloads. The resulting callable type +itself will be overloaded. + + :: + + class MyClass: + @overload + def __init__(self, x: int) -> None: ... + @overload + def __init__(self, x: str) -> None: ... + + reveal_type(accepts_callable(MyClass)) # overload of ``def (x: int) -> MyClass`` and ``def (x: str) -> MyClass`` + + +If the class is generic, the synthesized callable should include any class-scoped +type parameters that appear within the signature, but these type parameters should +be converted to function-scoped type parameters for the callable. +Any function-scoped type parameters in the ``__init__`` or ``__new__`` +method should also be included as function-scoped type parameters in the synthesized +callable. + + :: + + class MyClass[T]: + def __init__[V](self, x: T, y: list[V], z: V) -> None: ... + + reveal_type(accepts_callable(MyClass)) # ``def [T, V] (x: T, y: list[V], z: V) -> MyClass[T]`` + + diff --git a/docs/spec/index.rst b/docs/spec/index.rst index 6da9710e..3e77898c 100644 --- a/docs/spec/index.rst +++ b/docs/spec/index.rst @@ -17,6 +17,7 @@ Specification for the Python type system literal protocol callables + constructors overload dataclasses typeddict From 6694bd8e69cba0b82fc5c4683585696e261f8357 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 28 Mar 2024 09:01:31 -0600 Subject: [PATCH 02/11] Update docs/spec/constructors.rst Co-authored-by: Jelle Zijlstra --- docs/spec/constructors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index 925692eb..893764b9 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -254,7 +254,7 @@ Constructor Calls for type[T] When a value of type ``type[T]`` (where ``T`` is a type variable or a concrete class) is called, a type checker should evaluate the constructor call as if -it is being made on the class ``T`` (or the class the represents the upper bound +it is being made on the class ``T`` (or the class that represents the upper bound of type variable ``T``). This means the type checker should use ``__call__`` method of ``T``'s metaclass and the ``__new__`` and ``__init__`` methods of ``T`` to evaluate the constructor call. From aed835fd58ddd6625e54b5d1c969321c4486ac97 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 28 Mar 2024 09:02:08 -0600 Subject: [PATCH 03/11] Update docs/spec/constructors.rst Co-authored-by: Jelle Zijlstra --- docs/spec/constructors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index 893764b9..f5b86a24 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -255,7 +255,7 @@ Constructor Calls for type[T] When a value of type ``type[T]`` (where ``T`` is a type variable or a concrete class) is called, a type checker should evaluate the constructor call as if it is being made on the class ``T`` (or the class that represents the upper bound -of type variable ``T``). This means the type checker should use ``__call__`` +of type variable ``T``). This means the type checker should use the ``__call__`` method of ``T``'s metaclass and the ``__new__`` and ``__init__`` methods of ``T`` to evaluate the constructor call. From 983d6e32b096a515ad3e426c9e9ac648f9d5dd6b Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 28 Mar 2024 09:02:21 -0600 Subject: [PATCH 04/11] Update docs/spec/constructors.rst Co-authored-by: Jelle Zijlstra --- docs/spec/constructors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index f5b86a24..ebf218b5 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -259,7 +259,7 @@ of type variable ``T``). This means the type checker should use the ``__call__`` method of ``T``'s metaclass and the ``__new__`` and ``__init__`` methods of ``T`` to evaluate the constructor call. -It should be noted that such code could be unsafe because The type ``type[T]`` +It should be noted that such code could be unsafe because the type ``type[T]`` may represent subclasses of ``T``, and those subclasses could redefine the ``__new__`` and ``__init__`` methods in a way that is incompatible with the base class. Likewise, the metaclass of ``T`` could redefine the ``__call__`` From 1c64a6ce012b7cdbc5cefd27d99f9d1a5eac4fc2 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 28 Mar 2024 13:14:51 -0600 Subject: [PATCH 05/11] Added section on signature consistency between `__new__` and `__init__`. --- docs/spec/constructors.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index ebf218b5..c7b39452 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -291,6 +291,23 @@ these method evaluations, they should take on their default values. assert_type(MyClass2(1), MyClass2[int, str]) +Consistency of ``__new__`` and ``__init__`` +------------------------------------------- + +Type checkers may optionally validate that the ``__new__`` and ``__init__`` +methods for a class have consistent signatures. + + :: + + class MyClass: + def __new__(cls) -> Self: + return super().__new__(cls) + + # Type error: __new__ and __init__ have inconsistent signatures + def __init__(self, x: str) -> None: + pass + + Converting a Constructor to Callable ------------------------------------ From c5dfb1fcc5bf5ef0d142d7ce50a4e7fb41b3041b Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 28 Mar 2024 13:27:13 -0600 Subject: [PATCH 06/11] Incorporated PR feedback. --- docs/spec/constructors.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index c7b39452..3b6d0899 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -49,12 +49,12 @@ calls are not supported for that class. ================== After the metaclass ``__call__`` method has been evaluated, a type checker -should evaluate the ``__new__`` method of the class using the supplied arguments. -If the class is generic and explicitly specialized, the type checker -should partially specialize the ``__new__`` method using the supplied type -arguments. If the class is not explicitly specialized, class-scoped type -variables should be solved using the supplied arguments passed to the constructor -call. +should evaluate the ``__new__`` method of the class (if applicable) using +the supplied arguments. If the class is generic and explicitly specialized, +the type checker should partially specialize the ``__new__`` method using the +supplied type arguments. If the class is not explicitly specialized, +class-scoped type variables should be solved using the supplied arguments +passed to the constructor call. :: @@ -157,8 +157,8 @@ that arise during binding. =================== After evaluating the ``__new__`` method, a type checker should evaluate the -``__init__`` method using the supplied arguments. If the class is -generic and explicitly specialized (or specialized via the ``__new__`` method +``__init__`` method (if applicable) using the supplied arguments. If the class +is generic and explicitly specialized (or specialized via the ``__new__`` method return type), the type checker should partially specialize the ``__init__`` method using the supplied type arguments. If the class is not explicitly specialized, class-scoped type variables should be solved using the supplied From 07b0d119156c1698da63c2b55579a723ba7c2e9e Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 28 Mar 2024 13:41:10 -0600 Subject: [PATCH 07/11] Added clarification based on question in forum. --- docs/spec/constructors.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index 3b6d0899..7d1e72f0 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -73,7 +73,8 @@ passed to the constructor call. If any class-scoped type variables are not solved when evaluating the ``__new__`` method call using the supplied arguments, these type variables should be left -unsolved, allowing the ``__init__`` method to be used to solve them. +unsolved, allowing the ``__init__`` method (if applicable) to be used to solve +them. :: From 7b68ca2707600dca7404dc915a73fb32c8245f81 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Sun, 31 Mar 2024 09:41:55 -0700 Subject: [PATCH 08/11] Incorporated feedback about callable conversion. Clarified behaviors when a class doesn't inherit a `__new__` or `__init__` from a class other than `object`. --- docs/spec/constructors.rst | 111 +++++++++++++++++++++++++++++++------ 1 file changed, 94 insertions(+), 17 deletions(-) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index 7d1e72f0..e03530a3 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -50,11 +50,14 @@ calls are not supported for that class. After the metaclass ``__call__`` method has been evaluated, a type checker should evaluate the ``__new__`` method of the class (if applicable) using -the supplied arguments. If the class is generic and explicitly specialized, -the type checker should partially specialize the ``__new__`` method using the -supplied type arguments. If the class is not explicitly specialized, -class-scoped type variables should be solved using the supplied arguments -passed to the constructor call. +the supplied arguments. This step should be skipped if the class does not +define a ``__new__`` method and does not inherit a ``__new__`` method from +a base class other than ``object``. + +If the class is generic and explicitly specialized, the type checker should +partially specialize the ``__new__`` method using the supplied type arguments. +If the class is not explicitly specialized, class-scoped type variables should +be solved using the supplied arguments passed to the constructor call. :: @@ -165,6 +168,10 @@ method using the supplied type arguments. If the class is not explicitly specialized, class-scoped type variables should be solved using the supplied arguments passed to the constructor call. +This step should be skipped if the class does not define an ``__init__`` method +and does not inherit an ``__init__`` method from a base class other than +``object``. + :: class MyClass[T]: @@ -250,6 +257,23 @@ within a type annotation for the ``self`` parameter in an ``__init__`` method. def __init__(self: "MyClass4[T2, T1]") -> None: ... +Classes Without ``__new__`` and ``__init__`` Methods +==================================================== + +If a class does not define a ``__new__`` method or ``__init__`` method and +does not inherit either of these methods from a base class other than +``object``, a type checker should evaluate the argument list using the +``__new__`` and ``__init__`` methods from the ``object`` class. + + :: + + class MyClass5: + pass + + MyClass5() # OK + MyClass5(1) # Type error + + Constructor Calls for type[T] ----------------------------- @@ -328,24 +352,52 @@ Class objects are callable, which means they are compatible with callable types. When converting a class to a callable type, a type checker should use the following rules: -1. If the class defines an ``__init__`` method or inherits an ``__init__`` - method from a base class other than ``object``, the callable type should be - synthesized from the parameters of the ``__init__`` method after it is bound - to the class instance. The return type of the synthesized callable should be - the class itself. - -2. If the class does not define or inherit an ``__init__`` method from a base - class other than the ``object`` class, the callable type should be synthesized - from the parameters of the ``__new__`` method after it is bound to the class. - The return type of the synthesized callable should come from the bound - ``__new__`` method. +1. If the class has a custom metaclass that defines a ``__call__`` method + that is annotated with a return type other than ``Any`` or a subclass of the + class being constructed, a type checker should assume that the metaclass + ``__call__`` method is overriding ``type.__call__`` in some special manner. + In this case, the callable should be synthesized from the parameters and return + type of the metaclass ``__call__`` method after it is bound to the class, + and the ``__new__`` or ``__init__`` methods (if present) should be ignored. + This is an uncommon case. In the more typical case where there is no custom + metaclass that overrides ``type.__call__`` in a special manner, the metaclass + ``__call__`` signature should be ignored for purposes of converting to a + callable type. + +2. If the class defines a ``__new__`` method or inherits a ``__new__`` method + from a base class other than ``object``, a type checker should synthesize a + callable from the parameters and return type of that method after it is bound + to the class. + +3. If the method in step 2 has a return type that is not ``Any`` or a subclass + of the class being constructed, the final callable type is based on the + result of step 2, and the conversion process is complete. This is consistent + with the runtime behavior of the ``type.__call__`` method. + +4. If the class defines an ``__init__`` method or inherits an ``__init__`` method + from a base class other than ``object``, a callable type should be synthesized + from the parameters of the ``__init__`` method after it is bound to the class + instance. The return type of this synthesized callable should be the class + itself. + +5. If step 2 and 4 both produce no result because the class does not define or + inherit a ``__new__`` or ``__init__`` method from a class other than ``object``, + the type checker should synthesize callable types from the ``__new__`` and + ``__init__`` methods for the ``object`` class. + +6. Steps 2, 4 and 5 will produce either one or two callable types. The final + result of the conversion process is the union of these types. This will + reflect the callable signatures of the applicable ``__new__`` and + ``__init__`` methods. :: class A: + """ No __new__ or __init__ """ pass class B: + """ __new__ and __init__ """ def __new__(cls, *args, **kwargs) -> Self: ... @@ -353,12 +405,37 @@ following rules: ... class C: + """ __new__ but no __init__ """ def __new__(cls, x: int) -> int: ... + class CustomMeta(type): + def __call__(cls) -> NoReturn: + raise NotImplemented("Class not constructable") + + class D(metaclass=CustomMeta): + """ Custom metaclass that overrides type.__call__ """ + def __new__(cls, *args, **kwargs) -> Self: + """ This __new__ is ignored for purposes of conversion """ + pass + + + class E: + """ __new__ that causes __init__ to be ignored """ + + def __new__(cls) -> A: + return A.__new__() + + def __init__(self, x: int) -> None: + """ This __init__ is ignored for purposes of conversion """ + ... + + reveal_type(accepts_callable(A)) # ``def () -> A`` - reveal_type(accepts_callable(B)) # ``def (x: int) -> B`` + reveal_type(accepts_callable(B)) # ``def (*args, **kwargs) -> B | def (x: int) -> B`` reveal_type(accepts_callable(C)) # ``def (x: int) -> int`` + reveal_type(accepts_callable(D)) # ``def () -> NoReturn`` + reveal_type(accepts_callable(E)) # ``def () -> A`` If the ``__init__`` or ``__new__`` method is overloaded, the callable From 252421d20542ae64d9a7fae10bd5f274782d60b4 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Wed, 3 Apr 2024 09:43:00 -0700 Subject: [PATCH 09/11] Tweaked the spec based on Jelle's feedback about a `__new__` method that returns `Any`. Also clarified what happens when a call to `__call__` or `__new__` evaluates to a union type. --- docs/spec/constructors.rst | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index e03530a3..b5ff8154 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -95,10 +95,11 @@ For most classes, the return type for ``__new__`` method is typically ``Self``, but other types are also allowed. For example, the ``__new__`` method may return an instance of a subclass or an instance of some completely unrelated class. -If the return type of the ``__new__`` method is specified as ``Any`` (or the -return type is unspecified and not inferred), a type checker should -assume that the return type is ``Self``, and it should proceed to evaluate the -``__init__`` method. +If the return type of the ``__new__`` method evaluates to ``Any`` or a union that +includes ``Any``, a type checker should proceed to evaluate the ``__init__`` +method as if the return type of ``__new__`` was ``Self``. However, the final +evaluated type of the constructor call should include ``Any`` in this case, +unioned with the type informed by the evaluation of the ``__init__`` call. :: @@ -109,12 +110,15 @@ assume that the return type is ``Self``, and it should proceed to evaluate the def __init__(self): pass - assert_type(MyClass(), MyClass) + # Constructor call evaluates to `Any` (from __new__ method) + # unioned with `MyClass` (from __init__ method). + assert_type(MyClass(), Any | MyClass) -If the ``__new__`` method returns a type that is not an instance of the class -being constructed (or a subclass thereof), a type checker should assume that -the ``__init__`` method will not be called. This is consistent with the runtime -behavior of the ``type.__call__`` method. +If the evaluated return type of ``__new__`` is not an instance of the class +being constructed (or a subclass thereof) or is a union that includes such +a class, a type checker should assume that the ``__init__`` method will not be +called. This is consistent with the runtime behavior of the ``type.__call__`` +method. :: @@ -369,16 +373,18 @@ following rules: callable from the parameters and return type of that method after it is bound to the class. -3. If the method in step 2 has a return type that is not ``Any`` or a subclass - of the class being constructed, the final callable type is based on the - result of step 2, and the conversion process is complete. This is consistent - with the runtime behavior of the ``type.__call__`` method. +3. If the return type of the method in step 2 evaluates to a type that is not a + subclass of the class being constructed (or a union that includes such a + class), the final callable type is based on the result of step 2, and the + conversion process is complete. The ``__init__`` method is ignored in this + case. This is consistent with the runtime behavior of the ``type.__call__`` + method. 4. If the class defines an ``__init__`` method or inherits an ``__init__`` method from a base class other than ``object``, a callable type should be synthesized from the parameters of the ``__init__`` method after it is bound to the class - instance. The return type of this synthesized callable should be the class - itself. + instance resulting from step 2. The return type of this synthesized callable + should be the concrete value of ``Self``. 5. If step 2 and 4 both produce no result because the class does not define or inherit a ``__new__`` or ``__init__`` method from a class other than ``object``, From 21f18a5a717f4ba545c8d48c370f4f7aaf0979f2 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 9 Apr 2024 09:32:35 -0700 Subject: [PATCH 10/11] Updated handling of `Any` return types for `__call__` and `__new__` methods to reflect suggestion from @rchen152 in [this post](https://discuss.python.org/t/draft-typing-spec-chapter-for-constructors/49744/22). --- docs/spec/constructors.rst | 77 ++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index b5ff8154..2b13afd9 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -25,12 +25,12 @@ method. If so, it should evaluate the call of this method using the supplied arguments. If the metaclass is ``type``, this step can be skipped. If the evaluated return type of the ``__call__`` method is something other than -``Any`` or an instance of the class being constructed, a type checker should +an instance of the class being constructed, a type checker should assume that the metaclass ``__call__`` method is overriding ``type.__call__`` in some special manner, and it should not attempt to evaluate the ``__new__`` or ``__init__`` methods on the class. For example, some metaclass ``__call__`` methods are annotated to return ``NoReturn`` to indicate that constructor -calls are not supported for that class. +calls are not supported for that class. :: @@ -44,6 +44,11 @@ calls are not supported for that class. assert_type(MyClass(), Never) +If no return type annotation is provided for ``__call__``, a type checker may +assume that it does not override ``type.__call__`` in a special manner and +proceed as though the return type is an instance of the type specified by +the ``cls`` parameter. + ``__new__`` Method ================== @@ -91,39 +96,38 @@ them. assert_type(MyClass(1), MyClass[int]) assert_type(MyClass(""), MyClass[str]) -For most classes, the return type for ``__new__`` method is typically ``Self``, -but other types are also allowed. For example, the ``__new__`` method may return -an instance of a subclass or an instance of some completely unrelated class. +For most classes, the return type for the ``__new__`` method is typically +``Self``, but other types are also allowed. For example, the ``__new__`` +method may return an instance of a subclass or an instance of some completely +unrelated class. -If the return type of the ``__new__`` method evaluates to ``Any`` or a union that -includes ``Any``, a type checker should proceed to evaluate the ``__init__`` -method as if the return type of ``__new__`` was ``Self``. However, the final -evaluated type of the constructor call should include ``Any`` in this case, -unioned with the type informed by the evaluation of the ``__init__`` call. +If the evaluated return type of ``__new__`` is not an instance of the class +being constructed (or a subclass thereof) or is a union that includes such +a class, a type checker should assume that the ``__init__`` method will not be +called. This is consistent with the runtime behavior of the ``type.__call__`` +method. :: class MyClass: - def __new__(cls, *args, **kwargs) -> Any: - return super().__new__(*args, **kwargs) + def __new__(cls) -> int: + return 0 - def __init__(self): + # The __init__ method will not be called in this case, so + # it should not be evaluated. + def __init__(self, x: int): pass - # Constructor call evaluates to `Any` (from __new__ method) - # unioned with `MyClass` (from __init__ method). - assert_type(MyClass(), Any | MyClass) + assert_type(MyClass(), int) -If the evaluated return type of ``__new__`` is not an instance of the class -being constructed (or a subclass thereof) or is a union that includes such -a class, a type checker should assume that the ``__init__`` method will not be -called. This is consistent with the runtime behavior of the ``type.__call__`` -method. +For purposes of this test, an explicit return type of ``Any`` (or a +union containing ``Any``) should be treated as a type that is not an instance +of the class being constructed. :: class MyClass: - def __new__(cls) -> int: + def __new__(cls) -> Any: return 0 # The __init__ method will not be called in this case, so @@ -131,7 +135,11 @@ method. def __init__(self, x: int): pass - assert_type(MyClass(), int) + assert_type(MyClass(), Any) + +If the return type of ``__new__`` is not annotated, a type checker may assume +that the return type is ``Self`` and proceed with the assumption that the +``__init__`` method will be called. If the class is generic, it is possible for a ``__new__`` method to override the specialized class type and return a class instance that is specialized @@ -357,16 +365,19 @@ When converting a class to a callable type, a type checker should use the following rules: 1. If the class has a custom metaclass that defines a ``__call__`` method - that is annotated with a return type other than ``Any`` or a subclass of the - class being constructed, a type checker should assume that the metaclass - ``__call__`` method is overriding ``type.__call__`` in some special manner. - In this case, the callable should be synthesized from the parameters and return - type of the metaclass ``__call__`` method after it is bound to the class, - and the ``__new__`` or ``__init__`` methods (if present) should be ignored. - This is an uncommon case. In the more typical case where there is no custom - metaclass that overrides ``type.__call__`` in a special manner, the metaclass - ``__call__`` signature should be ignored for purposes of converting to a - callable type. + that is annotated with a return type other than a subclass of the + class being constructed (or a union that contains such a type), a type + checker should assume that the metaclass ``__call__`` method is overriding + ``type.__call__`` in some special manner. In this case, the callable should + be synthesized from the parameters and return type of the metaclass + ``__call__`` method after it is bound to the class, and the ``__new__`` or + ``__init__`` methods (if present) should be ignored. This is an uncommon + case. In the more typical case where there is no custom metaclass that + overrides ``type.__call__`` in a special manner, the metaclass ``__call__`` + signature should be ignored for purposes of converting to a callable type. + If a custom metaclass ``__call__`` method is present but does not have an + annotated return type, type checkers may assume that the method acts like + ``type.__call__`` and proceed to the next step. 2. If the class defines a ``__new__`` method or inherits a ``__new__`` method from a base class other than ``object``, a type checker should synthesize a From 0ec500a88ab746f4fbea0ee82d76493be9513383 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Wed, 10 Apr 2024 13:48:27 -0700 Subject: [PATCH 11/11] Incorporated feedback from @gvanrossum. --- docs/spec/constructors.rst | 45 +++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/docs/spec/constructors.rst b/docs/spec/constructors.rst index 2b13afd9..e0899678 100644 --- a/docs/spec/constructors.rst +++ b/docs/spec/constructors.rst @@ -9,7 +9,8 @@ Constructor Calls At runtime, a call to a class' constructor typically results in the invocation of three methods in the following order: 1. The ``__call__`` method of the metaclass (which is typically supplied by the - ``type`` class but can be overridden by a custom metaclass) + ``type`` class but can be overridden by a custom metaclass and which is + responsible for calling the next two methods) 2. The ``__new__`` static method of the class 3. The ``__init__`` instance method of the class @@ -24,8 +25,8 @@ class has a custom metaclass (a subclass of ``type``) that defines a ``__call__` method. If so, it should evaluate the call of this method using the supplied arguments. If the metaclass is ``type``, this step can be skipped. -If the evaluated return type of the ``__call__`` method is something other than -an instance of the class being constructed, a type checker should +If the evaluated return type of the ``__call__`` method indicates something +other than an instance of the class being constructed, a type checker should assume that the metaclass ``__call__`` method is overriding ``type.__call__`` in some special manner, and it should not attempt to evaluate the ``__new__`` or ``__init__`` methods on the class. For example, some metaclass ``__call__`` @@ -101,11 +102,13 @@ For most classes, the return type for the ``__new__`` method is typically method may return an instance of a subclass or an instance of some completely unrelated class. -If the evaluated return type of ``__new__`` is not an instance of the class -being constructed (or a subclass thereof) or is a union that includes such -a class, a type checker should assume that the ``__init__`` method will not be -called. This is consistent with the runtime behavior of the ``type.__call__`` -method. +If the evaluated return type of ``__new__`` is not the class being constructed +(or a subclass thereof), a type checker should assume that the ``__init__`` +method will not be called. This is consistent with the runtime behavior of +the ``type.__call__`` method. If the ``__new__`` method return type is +a union with one or more subtypes that are not instances of the class being +constructed (or a subclass thereof), a type checker should likewise assume that +the ``__init__`` method will not be called. :: @@ -113,8 +116,8 @@ method. def __new__(cls) -> int: return 0 - # The __init__ method will not be called in this case, so - # it should not be evaluated. + # In this case, the __init__ method should not be considered + # by the type checker when evaluating a constructor call. def __init__(self, x: int): pass @@ -156,8 +159,8 @@ with different type arguments. If the ``cls`` parameter within the ``__new__`` method is not annotated, type checkers should infer a type of ``type[Self]``. Regardless of whether the type of the ``cls`` parameter is explicit or inferred, the type checker should -bind the class being constructed to this parameter and report any type errors -that arise during binding. +bind the class being constructed to the ``cls`` parameter and report any type +errors that arise during binding. :: @@ -208,7 +211,7 @@ during binding. :: class MyClass[T]: - def __init__(self: "MyClass[int]") -> "None": ... + def __init__(self: "MyClass[int]") -> None: ... MyClass() # OK MyClass[int]() # OK @@ -216,10 +219,11 @@ during binding. The return type for ``__init__`` is always ``None``, which means the method cannot influence the return type of the constructor call by specifying -a return type. To work around this limitation, type checkers should allow -the ``self`` parameter to be annotated with a type that influences the resulting -type of the constructor call. This can be used in overloads to influence the -constructor return type for each overload. +a return type. There are cases where it is desirable for the ``__init__`` method +to influence the return type, especially when the ``__init__`` method is +overloaded. To enable this, type checkers should allow the ``self`` parameter +to be annotated with a type that influences the resulting type of the +constructor call. :: @@ -289,8 +293,8 @@ does not inherit either of these methods from a base class other than Constructor Calls for type[T] ----------------------------- -When a value of type ``type[T]`` (where ``T`` is a type variable or a concrete -class) is called, a type checker should evaluate the constructor call as if +When a value of type ``type[T]`` (where ``T`` is a concrete class or a type +variable) is called, a type checker should evaluate the constructor call as if it is being made on the class ``T`` (or the class that represents the upper bound of type variable ``T``). This means the type checker should use the ``__call__`` method of ``T``'s metaclass and the ``__new__`` and ``__init__`` methods of ``T`` @@ -362,7 +366,8 @@ Class objects are callable, which means they are compatible with callable types. reveal_type(accepts_callable(MyClass)) # ``def (x: int) -> MyClass`` When converting a class to a callable type, a type checker should use the -following rules: +following rules, which reflect the same rules specified above for evaluating +constructor calls: 1. If the class has a custom metaclass that defines a ``__call__`` method that is annotated with a return type other than a subclass of the