From abadb185fa253e3fe6b983a70d434a1662366eee Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 23 Feb 2024 22:29:07 -0800 Subject: [PATCH 1/5] Implement PEPs 705 and 728 --- pyanalyze/annotations.py | 70 ++++++++-- pyanalyze/arg_spec.py | 8 +- pyanalyze/error_code.py | 2 + pyanalyze/implementation.py | 162 ++++++++++++++++------ pyanalyze/signature.py | 11 +- pyanalyze/test_name_check_visitor.py | 44 ++++-- pyanalyze/test_typeddict.py | 42 ++++-- pyanalyze/typeshed.py | 21 ++- pyanalyze/value.py | 197 ++++++++++++++++++++++----- 9 files changed, 428 insertions(+), 129 deletions(-) diff --git a/pyanalyze/annotations.py b/pyanalyze/annotations.py index ab8714fd..6a97a471 100644 --- a/pyanalyze/annotations.py +++ b/pyanalyze/annotations.py @@ -51,7 +51,7 @@ import qcore -from typing_extensions import ParamSpec, TypedDict, get_origin, get_args +from typing_extensions import Literal, ParamSpec, TypedDict, get_origin, get_args from pyanalyze.annotated_types import get_annotated_types_extension @@ -87,6 +87,7 @@ TypeAlias, TypeAliasValue, TypeIsExtension, + TypedDictEntry, annotate_value, AnnotatedValue, AnySource, @@ -434,20 +435,32 @@ def _type_from_runtime( # mypy_extensions.TypedDict elif is_instance_of_typing_name(val, "_TypedDictMeta"): required_keys = getattr(val, "__required_keys__", None) + readonly_keys = getattr(val, "__readonly_keys__", None) # 3.8's typing.TypedDict doesn't have __required_keys__. With # inheritance, this makes it apparently impossible to figure out which # keys are required at runtime. total = getattr(val, "__total__", True) + extra_keys = None if hasattr(val, "__extra_keys__"): - extra_keys = _type_from_runtime(val.__extra_keys__, ctx) - else: - extra_keys = None + extra_keys = _type_from_runtime(val.__extra_keys__, ctx, is_typeddict=True) + if hasattr(val, "__closed__") and val.__closed__: + extra_keys = _type_from_runtime(val.__extra_items__, ctx, is_typeddict=True) + extra_readonly = False + while isinstance(extra_keys, TypeQualifierValue): + if extra_keys.qualifier == "ReadOnly": + extra_readonly = True + else: + ctx.show_error(f"{extra_keys.qualifier} not allowed on extra_keys") + extra_keys = extra_keys.value return TypedDictValue( { - key: _get_typeddict_value(value, ctx, key, required_keys, total) + key: _get_typeddict_value( + value, ctx, key, required_keys, total, readonly_keys + ) for key, value in val.__annotations__.items() }, extra_keys=extra_keys, + extra_keys_readonly=extra_readonly, ) elif val is InitVar: # On 3.6 and 3.7, InitVar[T] just returns InitVar at runtime, so we can't @@ -626,15 +639,26 @@ def _get_typeddict_value( key: str, required_keys: Optional[Container[str]], total: bool, -) -> Tuple[bool, Value]: + readonly_keys: Optional[Container[str]], +) -> TypedDictEntry: val = _type_from_runtime(value, ctx, is_typeddict=True) - if isinstance(val, Pep655Value): - return (val.required, val.value) if required_keys is None: required = total else: required = key in required_keys - return required, val + if readonly_keys is None: + readonly = False + else: + readonly = key in readonly_keys + while isinstance(val, TypeQualifierValue): + if val.qualifier == "ReadOnly": + readonly = True + elif val.qualifier == "Required": + required = True + elif val.qualifier == "NotRequired": + required = False + val = val.value + return TypedDictEntry(required=required, readonly=readonly, typ=val) def _eval_forward_ref( @@ -799,7 +823,7 @@ def _type_from_subscripted_value( if len(members) != 1: ctx.show_error("Required[] requires a single argument") return AnyValue(AnySource.error) - return Pep655Value(True, _type_from_value(members[0], ctx)) + return TypeQualifierValue("Required", _type_from_value(members[0], ctx)) elif is_typing_name(root, "NotRequired"): if not is_typeddict: ctx.show_error("NotRequired[] used in unsupported context") @@ -807,7 +831,15 @@ def _type_from_subscripted_value( if len(members) != 1: ctx.show_error("NotRequired[] requires a single argument") return AnyValue(AnySource.error) - return Pep655Value(False, _type_from_value(members[0], ctx)) + return TypeQualifierValue("NotRequired", _type_from_value(members[0], ctx)) + elif is_typing_name(root, "ReadOnly"): + if not is_typeddict: + ctx.show_error("ReadOnly[] used in unsupported context") + return AnyValue(AnySource.error) + if len(members) != 1: + ctx.show_error("ReadOnly[] requires a single argument") + return AnyValue(AnySource.error) + return TypeQualifierValue("ReadOnly", _type_from_value(members[0], ctx)) elif is_typing_name(root, "Unpack"): if not allow_unpack: ctx.show_error("Unpack[] used in unsupported context") @@ -921,8 +953,8 @@ class _SubscriptedValue(Value): @dataclass -class Pep655Value(Value): - required: bool +class TypeQualifierValue(Value): + qualifier: Literal["Required", "NotRequired", "ReadOnly"] value: Value @@ -1212,7 +1244,7 @@ def _value_of_origin_args( if len(args) != 1: ctx.show_error("Required[] requires a single argument") return AnyValue(AnySource.error) - return Pep655Value(True, _type_from_runtime(args[0], ctx)) + return TypeQualifierValue("Required", _type_from_runtime(args[0], ctx)) elif is_typing_name(origin, "NotRequired"): if not is_typeddict: ctx.show_error("NotRequired[] used in unsupported context") @@ -1220,7 +1252,15 @@ def _value_of_origin_args( if len(args) != 1: ctx.show_error("NotRequired[] requires a single argument") return AnyValue(AnySource.error) - return Pep655Value(False, _type_from_runtime(args[0], ctx)) + return TypeQualifierValue("NotRequired", _type_from_runtime(args[0], ctx)) + elif is_typing_name(origin, "ReadOnly"): + if not is_typeddict: + ctx.show_error("ReadOnly[] used in unsupported context") + return AnyValue(AnySource.error) + if len(args) != 1: + ctx.show_error("ReadOnly[] requires a single argument") + return AnyValue(AnySource.error) + return TypeQualifierValue("ReadOnly", _type_from_runtime(args[0], ctx)) elif is_typing_name(origin, "Unpack"): if not allow_unpack: ctx.show_error("Invalid usage of Unpack") diff --git a/pyanalyze/arg_spec.py b/pyanalyze/arg_spec.py index f703621a..0d82bf67 100644 --- a/pyanalyze/arg_spec.py +++ b/pyanalyze/arg_spec.py @@ -88,6 +88,7 @@ make_coro_type, NewTypeValue, SubclassValue, + TypedDictEntry, TypedDictValue, TypedValue, TypeVarValue, @@ -221,6 +222,7 @@ class ClassesSafeToInstantiate(PyObjectSequenceOption[type]): Value, Extension, KVPair, + TypedDictEntry, asynq.ConstFuture, range, tuple, @@ -735,10 +737,10 @@ def _uncached_get_argspec( SigParameter( key, ParameterKind.KEYWORD_ONLY, - default=None if required else KnownValue(...), - annotation=value, + default=None if entry.required else KnownValue(...), + annotation=entry.typ, ) - for key, (required, value) in td_type.items.items() + for key, entry in td_type.items.items() ] if td_type.extra_keys is not None: annotation = GenericValue( diff --git a/pyanalyze/error_code.py b/pyanalyze/error_code.py index 13dc0653..e8356fdb 100644 --- a/pyanalyze/error_code.py +++ b/pyanalyze/error_code.py @@ -109,6 +109,7 @@ class ErrorCode(enum.Enum): disallowed_import = 89 typeis_must_be_subtype = 90 invalid_typeguard = 91 + readonly_typeddict = 92 # Allow testing unannotated functions without too much fuss @@ -245,6 +246,7 @@ class ErrorCode(enum.Enum): "TypeIs narrowed type must be a subtype of the input type" ), ErrorCode.invalid_typeguard: "Invalid use of TypeGuard or TypeIs", + ErrorCode.readonly_typeddict: "TypedDict is read-only", } diff --git a/pyanalyze/implementation.py b/pyanalyze/implementation.py index f2c0e1b3..06c2ca1a 100644 --- a/pyanalyze/implementation.py +++ b/pyanalyze/implementation.py @@ -87,7 +87,7 @@ def clean_up_implementation_fn_return( - return_value: Union[Value, ImplReturn] + return_value: Union[Value, ImplReturn], ) -> ImplReturn: if isinstance(return_value, Value): return ImplReturn(return_value) @@ -494,7 +494,40 @@ def _sequence_getitem_impl(ctx: CallContext) -> ImplReturn: def _typeddict_setitem( self_value: TypedDictValue, key: Value, value: Value, ctx: CallContext ) -> None: - if not isinstance(key, KnownValue) or not isinstance(key.val, str): + if not isinstance(key, KnownValue): + if not TypedValue(str).is_assignable(key, ctx.visitor): + ctx.show_error( + f"TypedDict key must be str, not {key}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) + return + if self_value.extra_keys is None or self_value.extra_keys is NO_RETURN_VALUE: + ctx.show_error( + f"Cannot set unknown key {key} in TypedDict {self_value}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) + for td_key, entry in self_value.items.items(): + if not key.is_assignable(KnownValue(td_key), ctx.visitor): + continue + if entry.readonly: + ctx.show_error( + f"Cannot set readonly key {key} in TypedDict {self_value}", + ErrorCode.readonly_typeddict, + arg="k", + ) + can_assign = entry.typ.can_assign(value, ctx.visitor) + if isinstance(can_assign, CanAssignError): + ctx.show_error( + f"Value for key {key} must be {entry.typ}, not {value}", + ErrorCode.incompatible_argument, + arg="v", + detail=str(can_assign), + ) + return + + if not isinstance(key.val, str): ctx.show_error( f"TypedDict key must be a string literal (got {key})", ErrorCode.invalid_typeddict_key, @@ -502,6 +535,13 @@ def _typeddict_setitem( ) return if key.val not in self_value.items: + if self_value.extra_keys_readonly: + ctx.show_error( + f"Cannot set unknown key {key.val!r} in closed TypedDict {self_value}", + ErrorCode.readonly_typeddict, + arg="k", + ) + return if self_value.extra_keys is None: ctx.show_error( f"Key {key.val!r} does not exist in {self_value}", @@ -512,7 +552,15 @@ def _typeddict_setitem( else: expected_type = self_value.extra_keys else: - _, expected_type = self_value.items[key.val] + entry = self_value.items[key.val] + if entry.readonly: + ctx.show_error( + f"Cannot set readonly key {key.val!r} in TypedDict {self_value}", + ErrorCode.readonly_typeddict, + arg="k", + ) + return + expected_type = entry.typ tv_map = expected_type.can_assign(value, ctx.visitor) if isinstance(tv_map, CanAssignError): ctx.show_error( @@ -573,25 +621,29 @@ def inner(key: Value) -> Value: return AnyValue(AnySource.error) elif isinstance(key, KnownValue): try: - _, value = self_value.items[key.val] - return value + entry = self_value.items[key.val] + return entry.typ # probably KeyError, but catch anything in case it's an # unhashable str subclass or something except Exception: - if self_value.extra_keys is None: - ctx.show_error( - f"Unknown TypedDict key {key}", - ErrorCode.invalid_typeddict_key, - arg="k", - ) - return AnyValue(AnySource.error) - if self_value.extra_keys is not None: + pass + if ( + self_value.extra_keys is not None + and self_value.extra_keys is not NO_RETURN_VALUE + ): return self_value.extra_keys - ctx.show_error( - f"TypedDict key must be a literal, not {key}", - ErrorCode.invalid_typeddict_key, - arg="k", - ) + if isinstance(key, KnownValue): + ctx.show_error( + f"Unknown TypedDict key {key.val!r}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) + else: + ctx.show_error( + f"TypedDict key must be a literal, not {key}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) return AnyValue(AnySource.error) elif isinstance(self_value, DictIncompleteValue): val = self_value.get_value(key, ctx.visitor) @@ -646,29 +698,33 @@ def inner(key: Value) -> Value: return AnyValue(AnySource.error) elif isinstance(key, KnownValue): try: - required, value = self_value.items[key.val] + entry = self_value.items[key.val] # probably KeyError, but catch anything in case it's an # unhashable str subclass or something except Exception: - if self_value.extra_keys is None: - ctx.show_error( - f"Unknown TypedDict key {key.val!r}", - ErrorCode.invalid_typeddict_key, - arg="k", - ) - return AnyValue(AnySource.error) + pass else: - if required: - return value + if entry.required: + return entry.typ else: - return value | default - if self_value.extra_keys is not None: + return entry.typ | default + if ( + self_value.extra_keys is not None + and self_value.extra_keys is not NO_RETURN_VALUE + ): return self_value.extra_keys | default - ctx.show_error( - f"TypedDict key must be a literal, not {key}", - ErrorCode.invalid_typeddict_key, - arg="k", - ) + if isinstance(key, KnownValue): + ctx.show_error( + f"Unknown TypedDict key {key.val!r}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) + else: + ctx.show_error( + f"TypedDict key must be a literal, not {key}", + ErrorCode.invalid_typeddict_key, + arg="k", + ) return AnyValue(AnySource.error) elif isinstance(self_value, DictIncompleteValue): val = self_value.get_value(key, ctx.visitor) @@ -712,20 +768,29 @@ def _dict_pop_impl(ctx: CallContext) -> ImplReturn: return ImplReturn(AnyValue(AnySource.error)) elif isinstance(key, KnownValue): try: - is_required, expected_type = self_value.items[key.val] + entry = self_value.items[key.val] # probably KeyError, but catch anything in case it's an # unhashable str subclass or something except Exception: pass else: - if is_required: + if entry.required: ctx.show_error( f"Cannot pop required TypedDict key {key}", error_code=ErrorCode.incompatible_argument, arg="key", ) - return ImplReturn(_maybe_unite(expected_type, default)) - if self_value.extra_keys is not None: + if entry.readonly: + ctx.show_error( + f"Cannot pop readonly TypedDict key {key}", + error_code=ErrorCode.readonly_typeddict, + arg="key", + ) + return ImplReturn(_maybe_unite(entry.typ, default)) + if ( + self_value.extra_keys is not None + and self_value.extra_keys is not NO_RETURN_VALUE + ): return ImplReturn(_maybe_unite(self_value.extra_keys, default)) ctx.show_error( f"Key {key} does not exist in TypedDict", @@ -798,22 +863,31 @@ def _dict_setdefault_impl(ctx: CallContext) -> ImplReturn: return ImplReturn(AnyValue(AnySource.error)) elif isinstance(key, KnownValue): try: - _, expected_type = self_value.items[key.val] + entry = self_value.items[key.val] # probably KeyError, but catch anything in case it's an # unhashable str subclass or something except Exception: pass else: - tv_map = expected_type.can_assign(default, ctx.visitor) + if entry.readonly: + ctx.show_error( + f"Cannot setdefault readonly TypedDict key {key}", + error_code=ErrorCode.readonly_typeddict, + arg="key", + ) + tv_map = entry.typ.can_assign(default, ctx.visitor) if isinstance(tv_map, CanAssignError): ctx.show_error( f"TypedDict key {key.val} expected value of type" - f" {expected_type}, not {default}", + f" {entry.typ}, not {default}", ErrorCode.incompatible_argument, arg="default", ) - return ImplReturn(expected_type) - if self_value.extra_keys is not None: + return ImplReturn(entry.typ) + if ( + self_value.extra_keys is not None + and self_value.extra_keys is not NO_RETURN_VALUE + ): return ImplReturn(self_value.extra_keys | default) ctx.show_error( f"Key {key} does not exist in TypedDict", diff --git a/pyanalyze/signature.py b/pyanalyze/signature.py index e54f7d35..1a48750c 100644 --- a/pyanalyze/signature.py +++ b/pyanalyze/signature.py @@ -65,6 +65,7 @@ from .value import ( SelfT, TypeIsExtension, + TypedDictEntry, annotate_value, AnnotatedValue, AnySource, @@ -1027,7 +1028,9 @@ def bind_arguments( ) in actual_args.keywords.items(): if key in keywords_consumed: continue - items[key] = (definitely_provided, composite.value) + items[key] = TypedDictEntry( + composite.value, required=definitely_provided + ) position = KWARGS if actual_args.ellipsis: star_kwargs_value = GenericValue( @@ -1819,12 +1822,12 @@ def make( elif param.kind is ParameterKind.VAR_KEYWORD and isinstance( param.annotation, TypedDictValue ): - for name, (is_required, value) in param.annotation.items.items(): + for name, entry in param.annotation.items.items(): param_dict[name] = SigParameter( name, ParameterKind.KEYWORD_ONLY, - annotation=value, - default=None if is_required else AnyValue(AnySource.marker), + annotation=entry.typ, + default=None if entry.required else AnyValue(AnySource.marker), ) i += 1 if param.annotation.extra_keys is not None: diff --git a/pyanalyze/test_name_check_visitor.py b/pyanalyze/test_name_check_visitor.py index 25fbd131..630eebcd 100644 --- a/pyanalyze/test_name_check_visitor.py +++ b/pyanalyze/test_name_check_visitor.py @@ -42,6 +42,7 @@ ReferencingValue, SequenceValue, SubclassValue, + TypedDictEntry, TypedDictValue, TypedValue, TypeVarValue, @@ -133,6 +134,7 @@ def _make_module(code_str: str) -> types.ModuleType: CallableValue=CallableValue, DictIncompleteValue=DictIncompleteValue, KVPair=KVPair, + TypedDictEntry=TypedDictEntry, GenericValue=GenericValue, KnownValue=KnownValue, MultiValuedValue=MultiValuedValue, @@ -908,11 +910,13 @@ def test(self, uid: Uid): class TestImports(TestNameCheckVisitorBase): def test_star_import(self): - self.assert_passes(""" + self.assert_passes( + """ from qcore.asserts import * assert_eq(1, 1) - """) + """ + ) @assert_passes() def test_local_import(self): @@ -1377,7 +1381,8 @@ def run_and_get_call_map(self, code_str, **kwargs): return collector.map def test_member_function_call(self): - call_map = self.run_and_get_call_map(""" + call_map = self.run_and_get_call_map( + """ class TestClass(object): def __init__(self): self.first_function(5) @@ -1388,7 +1393,8 @@ def first_function(self, x): def second_function(self, y, z): print(y + z) - """) + """ + ) assert "TestClass.first_function" in call_map["TestClass.second_function"] assert "TestClass.__init__" in call_map["TestClass.first_function"] @@ -2001,7 +2007,8 @@ def f(self, value: bool) -> None: # E: incompatible_override class TestWalrus(TestNameCheckVisitorBase): def test(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import Optional def opt() -> Optional[int]: @@ -2015,10 +2022,12 @@ def capybara(): if (y := opt()) is not None: assert_is_value(y, TypedValue(int)) assert_is_value(y, TypedValue(int) | KnownValue(None)) - """) + """ + ) def test_and(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import Optional def opt() -> Optional[int]: @@ -2028,31 +2037,38 @@ def capybara(cond): if (x := opt()) and cond: assert_is_value(x, TypedValue(int)) assert_is_value(x, TypedValue(int) | KnownValue(None)) - """) - self.assert_passes(""" + """ + ) + self.assert_passes( + """ from typing import Set def func(myvar: str, strset: Set[str]) -> None: if (encoder_type := myvar) and myvar in strset: print(encoder_type) - """) + """ + ) def test_if_exp(self): - self.assert_passes(""" + self.assert_passes( + """ def capybara(cond): (x := 2) if cond else (x := 1) assert_is_value(x, KnownValue(2) | KnownValue(1)) - """) + """ + ) def test_comprehension_scope(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import List, Optional def capybara(elts: List[Optional[int]]) -> None: if any((x := i) is not None for i in elts): assert_is_value(x, TypedValue(int) | KnownValue(None)) print(i) # E: undefined_name - """) + """ + ) class TestUnion(TestNameCheckVisitorBase): diff --git a/pyanalyze/test_typeddict.py b/pyanalyze/test_typeddict.py index 547d0468..4d28e42e 100644 --- a/pyanalyze/test_typeddict.py +++ b/pyanalyze/test_typeddict.py @@ -2,7 +2,7 @@ from .implementation import assert_is_value from .test_name_check_visitor import TestNameCheckVisitorBase from .test_node_visitor import assert_passes -from .value import TypedDictValue, TypedValue, AnyValue, AnySource +from .value import TypedDictValue, TypedValue, AnyValue, AnySource, TypedDictEntry class TestExtraKeys(TestNameCheckVisitorBase): @@ -20,7 +20,7 @@ def capybara() -> None: assert_is_value( x, TypedDictValue( - {"a": (True, TypedValue(str))}, extra_keys=TypedValue(int) + {"a": TypedDictEntry(TypedValue(str))}, extra_keys=TypedValue(int) ), ) @@ -74,7 +74,7 @@ def capybara() -> None: @assert_passes() def test_compatibility(self): from pyanalyze.extensions import has_extra_keys - from typing_extensions import TypedDict + from typing_extensions import ReadOnly, TypedDict from typing import Any, Dict @has_extra_keys(int) @@ -89,15 +89,27 @@ class TD2(TypedDict): class TD3(TypedDict): a: str + @has_extra_keys(ReadOnly[int]) + class TD4(TypedDict): + a: str + def want_td(td: TD) -> None: pass + def want_td4(td: TD4) -> None: + pass + def capybara(td: TD, td2: TD2, td3: TD3, anydict: Dict[str, Any]) -> None: want_td(td) - want_td(td2) + want_td(td2) # E: incompatible_argument want_td(td3) # E: incompatible_argument want_td(anydict) + want_td4(td) + want_td4(td2) + want_td4(td3) # E: incompatible_argument + want_td4(anydict) + @assert_passes() def test_iteration(self): from pyanalyze.extensions import has_extra_keys @@ -122,7 +134,7 @@ def capybara(td: TD, td2: TD2) -> None: assert_type(k, str) assert_type(v, str) for k in td2: - assert_type(k, Literal["a"]) + assert_type(k, Union[str, Literal["a"]]) class TestTypedDict(TestNameCheckVisitorBase): @@ -143,7 +155,10 @@ def capybara(): assert_is_value( cap, TypedDictValue( - {"x": (True, TypedValue(int)), "y": (True, TypedValue(str))} + { + "x": TypedDictEntry(TypedValue(int)), + "y": TypedDictEntry(TypedValue(str)), + } ), ) Capybara(x=1) # E: incompatible_call @@ -152,7 +167,10 @@ def capybara(): assert_is_value( maybe_cap, TypedDictValue( - {"x": (True, TypedValue(int)), "y": (False, TypedValue(str))} + { + "x": TypedDictEntry(TypedValue(int)), + "y": TypedDictEntry(TypedValue(str), required=False), + } ), ) @@ -179,14 +197,20 @@ def capybara(x: T, y: T2): assert_is_value( x, TypedDictValue( - {"a": (True, TypedValue(int)), "b": (True, TypedValue(str))} + { + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(str)), + } ), ) assert_is_value(x["a"], TypedValue(int)) assert_is_value( y, TypedDictValue( - {"a": (True, TypedValue(int)), "b": (True, TypedValue(str))} + { + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(str)), + } ), ) assert_is_value(y["a"], TypedValue(int)) diff --git a/pyanalyze/typeshed.py b/pyanalyze/typeshed.py index b2355ecf..1a4a92fa 100644 --- a/pyanalyze/typeshed.py +++ b/pyanalyze/typeshed.py @@ -40,9 +40,9 @@ from .analysis_lib import is_positional_only_arg_name from .annotations import ( Context, + TypeQualifierValue, make_type_var_value, DecoratorValue, - Pep655Value, SyntheticEvaluator, type_from_value, value_from_ast, @@ -74,6 +74,7 @@ DeprecatedExtension, Extension, SyntheticModuleValue, + TypedDictEntry, annotate_value, extract_typevars, GenericValue, @@ -1103,6 +1104,7 @@ def _make_typeddict( total = True if isinstance(info.ast, ast.ClassDef): for keyword in info.ast.keywords: + # TODO support PEP 728 here if keyword.arg == "total": val = self._parse_expr(keyword.value, module) if isinstance(val, KnownValue) and isinstance(val.val, bool): @@ -1126,11 +1128,18 @@ def _make_typeddict( ) return TypedDictValue(items) - def _make_td_value(self, field: Value, total: bool) -> Tuple[bool, Value]: - if isinstance(field, Pep655Value): - return (field.required, field.value) - else: - return (total, field) + def _make_td_value(self, field: Value, total: bool) -> TypedDictEntry: + readonly = False + required = total + while isinstance(field, TypeQualifierValue): + if field.qualifier == "ReadOnly": + readonly = True + elif field.qualifier == "Required": + required = True + elif field.qualifier == "NotRequired": + required = False + field = field.value + return TypedDictEntry(readonly=readonly, required=required, typ=field) def _value_from_info( self, info: typeshed_client.resolver.ResolvedName, module: str diff --git a/pyanalyze/value.py b/pyanalyze/value.py index 636588ef..6454be99 100644 --- a/pyanalyze/value.py +++ b/pyanalyze/value.py @@ -956,8 +956,8 @@ def maybe_specify_error( if isinstance(can_assign, CanAssignError): return CanAssignError(f"In TypedDict key {key!r}", [can_assign]) elif i == 1: - for key, (_, value) in other.items.items(): - can_assign = expected.can_assign(value, ctx) + for key, entry in other.items.items(): + can_assign = expected.can_assign(entry.typ, ctx) if isinstance(can_assign, CanAssignError): return CanAssignError(f"In TypedDict key {key!r}", [can_assign]) elif isinstance(other, SequenceValue) and self.typ in { @@ -1241,22 +1241,42 @@ def get_value(self, key: Value, ctx: CanAssignContext) -> Value: return unite_values(*possible_values) +@dataclass(frozen=True) +class TypedDictEntry: + typ: Value + required: bool = True + readonly: bool = False + + def __str__(self) -> str: + val = str(self.typ) + if self.readonly: + val = f"Readonly[{val}]" + if not self.required: + val = f"NotRequired[{val}]" + return val + + @dataclass(init=False) class TypedDictValue(GenericValue): """Equivalent to ``typing.TypedDict``; a dictionary with a known set of string keys.""" - items: Dict[str, Tuple[bool, Value]] + items: Dict[str, TypedDictEntry] """The items of the ``TypedDict``. Required items are represented as (True, value) and optional ones as (False, value).""" extra_keys: Optional[Value] = None """The type of unknown keys, if any.""" + extra_keys_readonly: bool = False + """Whether the extra keys are readonly.""" def __init__( - self, items: Dict[str, Tuple[bool, Value]], extra_keys: Optional[Value] = None + self, + items: Dict[str, TypedDictEntry], + extra_keys: Optional[Value] = None, + extra_keys_readonly: bool = False, ) -> None: value_types = [] if items: - value_types += [val for _, val in items.values()] + value_types += [val.typ for val in items.values()] if extra_keys is not None: value_types.append(extra_keys) value_type = ( @@ -1269,97 +1289,206 @@ def __init__( super().__init__(dict, (key_type, value_type)) self.items = items self.extra_keys = extra_keys + self.extra_keys_readonly = extra_keys_readonly def num_required_keys(self) -> int: - return sum(1 for required, _ in self.items.values() if required) + return sum(1 for entry in self.items.values() if entry.required) def all_keys_required(self) -> bool: - return all(required for required, _ in self.items.values()) + return all(entry.required for entry in self.items.values()) def can_assign(self, other: Value, ctx: CanAssignContext) -> CanAssign: if isinstance(other, DictIncompleteValue): bounds_maps = [] - for key, (is_required, value) in self.items.items(): + for key, entry in self.items.items(): their_value = other.get_value(KnownValue(key), ctx) if their_value is UNINITIALIZED_VALUE: - if is_required: + if entry.required: return CanAssignError(f"Key {key} is missing in {other}") else: continue - can_assign = value.can_assign(their_value, ctx) + can_assign = entry.typ.can_assign(their_value, ctx) if isinstance(can_assign, CanAssignError): return CanAssignError( f"Types for key {key} are incompatible", children=[can_assign] ) bounds_maps.append(can_assign) + for pair in other.kv_pairs: + for key_type in flatten_values(pair.key, unwrap_annotated=True): + if isinstance(key_type, KnownValue): + if not isinstance(key_type.val, str): + return CanAssignError(f"Key {pair.key} is not a string") + if key_type.val not in self.items: + if self.extra_keys is NO_RETURN_VALUE: + return CanAssignError( + f"Key {key_type.val!r} is not allowed in closed TypedDict {self}" + ) + elif self.extra_keys is not None: + can_assign = self.extra_keys.can_assign(pair.value, ctx) + if isinstance(can_assign, CanAssignError): + return CanAssignError( + f"Type for extra key {pair.key} is incompatible", + children=[can_assign], + ) + bounds_maps.append(can_assign) + else: + can_assign = TypedValue(str).can_assign(key_type, ctx) + if isinstance(can_assign, CanAssignError): + return CanAssignError( + f"Type for key {pair.key} is not a string", + children=[can_assign], + ) + if self.extra_keys is NO_RETURN_VALUE: + return CanAssignError( + f"Key {pair.key} is not allowed in closed TypedDict {self}" + ) + elif self.extra_keys is not None: + can_assign = self.extra_keys.can_assign(pair.value, ctx) + if isinstance(can_assign, CanAssignError): + return CanAssignError( + f"Type for extra key {pair.key} is incompatible", + children=[can_assign], + ) + bounds_maps.append(can_assign) return unify_bounds_maps(bounds_maps) elif isinstance(other, TypedDictValue): bounds_maps = [] - for key, (is_required, value) in self.items.items(): + for key, entry in self.items.items(): if key not in other.items: - if is_required: - return CanAssignError(f"Key {key} is missing in {other}") + if entry.required: + return CanAssignError( + f"Required key {key} is missing in {other}" + ) + if not entry.readonly: + # "other" may be a subclass of its TypedDict type that sets a different key + return CanAssignError( + f"Mutable key {key} is missing in {other}" + ) + extra_keys_type = other.extra_keys or TypedValue(object) + can_assign = entry.typ.can_assign(extra_keys_type, ctx) + if isinstance(can_assign, CanAssignError): + return CanAssignError( + f"Type for key {key} is incompatible with extra keys type {extra_keys_type}", + children=[can_assign], + ) else: - can_assign = value.can_assign(other.items[key][1], ctx) + their_entry = other.items[key] + if entry.required and not their_entry.required: + return CanAssignError( + f"Required key {key} is non-required in {other}" + ) + if ( + not entry.required + and not entry.readonly + and their_entry.required + ): + # This means we may del the key, but the other TypedDict does not + # allow it + return CanAssignError( + f"Mutable key {key} is required in {other}" + ) + + can_assign = entry.typ.can_assign(their_entry.typ, ctx) if isinstance(can_assign, CanAssignError): return CanAssignError( f"Types for key {key} are incompatible", children=[can_assign], ) bounds_maps.append(can_assign) - # TODO: What if only one of the two has extra keys? - if self.extra_keys is not None and other.extra_keys is not None: - can_assign = self.extra_keys.can_assign(other.extra_keys, ctx) + if not entry.readonly: + can_assign = their_entry.typ.can_assign(entry.typ, ctx) + if isinstance(can_assign, CanAssignError): + return CanAssignError( + f"Types for mutable key {key} are incompatible", + children=[can_assign], + ) + bounds_maps.append(can_assign) + if not self.extra_keys_readonly and other.extra_keys_readonly: + return CanAssignError(f"Extra keys are readonly in {other}") + if self.extra_keys is not None: + their_extra_keys = other.extra_keys or TypedValue(object) + can_assign = self.extra_keys.can_assign(their_extra_keys, ctx) if isinstance(can_assign, CanAssignError): return CanAssignError( "Types for extra keys are incompatible", children=[can_assign] ) bounds_maps.append(can_assign) + if not self.extra_keys_readonly: + can_assign = their_extra_keys.can_assign(self.extra_keys, ctx) + if isinstance(can_assign, CanAssignError): + return CanAssignError( + "Types for mutable extra keys are incompatible", + children=[can_assign], + ) + bounds_maps.append(can_assign) return unify_bounds_maps(bounds_maps) elif isinstance(other, KnownValue) and isinstance(other.val, dict): bounds_maps = [] - for key, (is_required, value) in self.items.items(): + for key, entry in self.items.items(): if key not in other.val: - if is_required: + if entry.required: return CanAssignError(f"Key {key} is missing in {other}") else: - can_assign = value.can_assign(KnownValue(other.val[key]), ctx) + can_assign = entry.typ.can_assign(KnownValue(other.val[key]), ctx) if isinstance(can_assign, CanAssignError): return CanAssignError( f"Types for key {key} are incompatible", children=[can_assign], ) bounds_maps.append(can_assign) + for key, value in other.val.items(): + if key not in self.items: + if self.extra_keys is NO_RETURN_VALUE: + return CanAssignError( + f"Key {key} is not allowed in closed TypedDict {self}" + ) + elif self.extra_keys is not None: + can_assign = self.extra_keys.can_assign(KnownValue(value), ctx) + if isinstance(can_assign, CanAssignError): + return CanAssignError( + f"Type for extra key {key} is incompatible", + children=[can_assign], + ) + bounds_maps.append(can_assign) return unify_bounds_maps(bounds_maps) return super().can_assign(other, ctx) def substitute_typevars(self, typevars: TypeVarMap) -> "TypedDictValue": return TypedDictValue( { - key: (is_required, value.substitute_typevars(typevars)) - for key, (is_required, value) in self.items.items() + key: TypedDictEntry( + entry.typ.substitute_typevars(typevars), + required=entry.required, + readonly=entry.readonly, + ) + for key, entry in self.items.items() }, extra_keys=( self.extra_keys.substitute_typevars(typevars) if self.extra_keys is not None else None ), + extra_keys_readonly=self.extra_keys_readonly, ) def __str__(self) -> str: - items = [ - f'"{key}": {value if required else "NotRequired[" + str(value) + "]"}' - for key, (required, value) in self.items.items() - ] - return "TypedDict({%s})" % ", ".join(items) + entries = list(self.items.items()) + if self.extra_keys is not None and self.extra_keys is not NO_RETURN_VALUE: + extra_typ = str(self.extra_keys) + if self.extra_keys_readonly: + extra_typ = f"ReadOnly[{extra_typ}]" + entries.append(("__extra_items__", extra_typ)) + items = [f'"{key}": {entry}' for key, entry in entries] + closed = ", closed=True" if self.extra_keys is not None else "" + return f"TypedDict({{{', '.join(items)}}}{closed})" def __hash__(self) -> int: return hash(tuple(sorted(self.items))) def walk_values(self) -> Iterable["Value"]: yield self - for _, value in self.items.values(): - yield from value.walk_values() + for entry in self.items.values(): + yield from entry.typ.walk_values() @dataclass(unsafe_hash=True, init=False) @@ -2533,12 +2662,12 @@ def concrete_values_from_iterable( return value.args[0] return members elif isinstance(value, TypedDictValue): - if value.extra_keys is None and all( - required for required, _ in value.items.items() + if value.extra_keys is NO_RETURN_VALUE and all( + entry.required for entry in value.items.values() ): return [KnownValue(key) for key in value.items] possibilities = [KnownValue(key) for key in value.items] - if value.extra_keys is not None: + if value.extra_keys is not NO_RETURN_VALUE: possibilities.append(TypedValue(str)) return MultiValuedValue(possibilities) elif isinstance(value, DictIncompleteValue): @@ -2622,8 +2751,8 @@ def kv_pairs_from_mapping( return value_val.kv_pairs elif isinstance(value_val, TypedDictValue): pairs = [ - KVPair(KnownValue(key), value, is_required=required) - for key, (required, value) in value_val.items.items() + KVPair(KnownValue(key), entry.typ, is_required=entry.required) + for key, entry in value_val.items.items() ] if value_val.extra_keys is not None: pairs.append( From e59318553b7de800779bd3f1c03c599a4546939b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 23 Feb 2024 23:10:11 -0800 Subject: [PATCH 2/5] More fixes and tests --- docs/changelog.md | 2 + pyanalyze/annotated_types.py | 9 ++- pyanalyze/annotations.py | 24 ++++++-- pyanalyze/implementation.py | 111 +++++++++++++++++++++++++++++++++-- pyanalyze/signature.py | 9 ++- pyanalyze/test_typeddict.py | 102 ++++++++++++++++++++++++++++++++ pyanalyze/test_value.py | 37 +++++++++--- pyanalyze/value.py | 6 +- 8 files changed, 276 insertions(+), 24 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index f7c9a40d..a85063d4 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,8 @@ ## Unreleased +- Add support for the `ReadOnly` type qualifier (PEP 705) and + for the `closed=True` TypedDict argument (PEP 728) (#723) - Fix some higher-order behavior of `TypeGuard` and `TypeIs` (#719) - Add support for `TypeIs` from PEP 742 (#718) - More PEP 695 support: generic classes and functions. Scoping rules diff --git a/pyanalyze/annotated_types.py b/pyanalyze/annotated_types.py index 92cb276d..180bd6f6 100644 --- a/pyanalyze/annotated_types.py +++ b/pyanalyze/annotated_types.py @@ -13,6 +13,7 @@ from .extensions import CustomCheck from .value import ( + NO_RETURN_VALUE, AnnotatedValue, AnyValue, CanAssignError, @@ -293,7 +294,7 @@ def _min_len_of_value(val: Value) -> Optional[int]: elif isinstance(val, DictIncompleteValue): return sum(pair.is_required and not pair.is_many for pair in val.kv_pairs) elif isinstance(val, TypedDictValue): - return sum(required for required, _ in val.items.values()) + return sum(entry.required for entry in val.items.values()) else: return None @@ -314,6 +315,10 @@ def _max_len_of_value(val: Value) -> Optional[int]: if pair.is_required: maximum += 1 return maximum + elif isinstance(val, TypedDictValue): + if val.extra_keys is not NO_RETURN_VALUE: + # May have arbitrary number of extra keys + return None + return len(val.items) else: - # Always None for TypedDicts as TypedDicts may have arbitrary extra keys return None diff --git a/pyanalyze/annotations.py b/pyanalyze/annotations.py index 6a97a471..7531c407 100644 --- a/pyanalyze/annotations.py +++ b/pyanalyze/annotations.py @@ -823,7 +823,9 @@ def _type_from_subscripted_value( if len(members) != 1: ctx.show_error("Required[] requires a single argument") return AnyValue(AnySource.error) - return TypeQualifierValue("Required", _type_from_value(members[0], ctx)) + return TypeQualifierValue( + "Required", _type_from_value(members[0], ctx, is_typeddict=True) + ) elif is_typing_name(root, "NotRequired"): if not is_typeddict: ctx.show_error("NotRequired[] used in unsupported context") @@ -831,7 +833,9 @@ def _type_from_subscripted_value( if len(members) != 1: ctx.show_error("NotRequired[] requires a single argument") return AnyValue(AnySource.error) - return TypeQualifierValue("NotRequired", _type_from_value(members[0], ctx)) + return TypeQualifierValue( + "NotRequired", _type_from_value(members[0], ctx, is_typeddict=True) + ) elif is_typing_name(root, "ReadOnly"): if not is_typeddict: ctx.show_error("ReadOnly[] used in unsupported context") @@ -839,7 +843,9 @@ def _type_from_subscripted_value( if len(members) != 1: ctx.show_error("ReadOnly[] requires a single argument") return AnyValue(AnySource.error) - return TypeQualifierValue("ReadOnly", _type_from_value(members[0], ctx)) + return TypeQualifierValue( + "ReadOnly", _type_from_value(members[0], ctx, is_typeddict=True) + ) elif is_typing_name(root, "Unpack"): if not allow_unpack: ctx.show_error("Unpack[] used in unsupported context") @@ -1244,7 +1250,9 @@ def _value_of_origin_args( if len(args) != 1: ctx.show_error("Required[] requires a single argument") return AnyValue(AnySource.error) - return TypeQualifierValue("Required", _type_from_runtime(args[0], ctx)) + return TypeQualifierValue( + "Required", _type_from_runtime(args[0], ctx, is_typeddict=True) + ) elif is_typing_name(origin, "NotRequired"): if not is_typeddict: ctx.show_error("NotRequired[] used in unsupported context") @@ -1252,7 +1260,9 @@ def _value_of_origin_args( if len(args) != 1: ctx.show_error("NotRequired[] requires a single argument") return AnyValue(AnySource.error) - return TypeQualifierValue("NotRequired", _type_from_runtime(args[0], ctx)) + return TypeQualifierValue( + "NotRequired", _type_from_runtime(args[0], ctx, is_typeddict=True) + ) elif is_typing_name(origin, "ReadOnly"): if not is_typeddict: ctx.show_error("ReadOnly[] used in unsupported context") @@ -1260,7 +1270,9 @@ def _value_of_origin_args( if len(args) != 1: ctx.show_error("ReadOnly[] requires a single argument") return AnyValue(AnySource.error) - return TypeQualifierValue("ReadOnly", _type_from_runtime(args[0], ctx)) + return TypeQualifierValue( + "ReadOnly", _type_from_runtime(args[0], ctx, is_typeddict=True) + ) elif is_typing_name(origin, "Unpack"): if not allow_unpack: ctx.show_error("Invalid usage of Unpack") diff --git a/pyanalyze/implementation.py b/pyanalyze/implementation.py index 06c2ca1a..15758f74 100644 --- a/pyanalyze/implementation.py +++ b/pyanalyze/implementation.py @@ -508,6 +508,7 @@ def _typeddict_setitem( ErrorCode.invalid_typeddict_key, arg="k", ) + return for td_key, entry in self_value.items.items(): if not key.is_assignable(KnownValue(td_key), ctx.visitor): continue @@ -517,6 +518,7 @@ def _typeddict_setitem( ErrorCode.readonly_typeddict, arg="k", ) + return can_assign = entry.typ.can_assign(value, ctx.visitor) if isinstance(can_assign, CanAssignError): ctx.show_error( @@ -525,6 +527,7 @@ def _typeddict_setitem( arg="v", detail=str(can_assign), ) + return return if not isinstance(key.val, str): @@ -749,6 +752,88 @@ def inner(key: Value) -> Value: return flatten_unions(inner, ctx.vars["key"]) +def _dict_delitem_impl(ctx: CallContext) -> ImplReturn: + key = ctx.vars["key"] + varname = ctx.visitor.varname_for_self_constraint(ctx.node) + self_value = replace_known_sequence_value(ctx.vars["self"]) + + if not _check_dict_key_hashability(key, ctx, "key"): + return ImplReturn(AnyValue(AnySource.error)) + + if isinstance(self_value, TypedDictValue): + if not TypedValue(str).is_assignable(key, ctx.visitor): + ctx.show_error( + f"TypedDict key must be str, not {key}", + ErrorCode.invalid_typeddict_key, + arg="key", + ) + return ImplReturn(AnyValue(AnySource.error)) + elif isinstance(key, KnownValue): + try: + entry = self_value.items[key.val] + # probably KeyError, but catch anything in case it's an + # unhashable str subclass or something + except Exception: + pass + else: + if entry.required: + ctx.show_error( + f"Cannot delete required TypedDict key {key}", + error_code=ErrorCode.incompatible_argument, + arg="key", + ) + elif entry.readonly: + ctx.show_error( + f"Cannot delete readonly TypedDict key {key}", + error_code=ErrorCode.readonly_typeddict, + arg="key", + ) + return ImplReturn(KnownValue(None)) + if self_value.extra_keys_readonly: + ctx.show_error( + f"Cannot delete unknown key {key} in closed TypedDict {self_value}", + ErrorCode.readonly_typeddict, + arg="key", + ) + elif self_value.extra_keys is None or self_value.extra_keys is NO_RETURN_VALUE: + ctx.show_error( + f"Key {key} does not exist in TypedDict", + ErrorCode.invalid_typeddict_key, + arg="key", + ) + elif isinstance(self_value, DictIncompleteValue): + existing_value = self_value.get_value(key, ctx.visitor) + is_present = existing_value is not UNINITIALIZED_VALUE + if varname is not None and isinstance(key, KnownValue): + new_value = DictIncompleteValue( + self_value.typ, + [pair for pair in self_value.kv_pairs if pair.key != key], + ) + no_return_unless = Constraint( + varname, ConstraintType.is_value_object, True, new_value + ) + else: + no_return_unless = NULL_CONSTRAINT + if not is_present: + ctx.show_error( + f"Key {key} does not exist in dictionary {self_value}", + error_code=ErrorCode.incompatible_argument, + arg="key", + ) + return ImplReturn(KnownValue(None)) + return ImplReturn(KnownValue(None), no_return_unless=no_return_unless) + elif isinstance(self_value, TypedValue): + key_type = self_value.get_generic_arg_for_type(dict, ctx.visitor, 0) + tv_map = key_type.can_assign(key, ctx.visitor) + if isinstance(tv_map, CanAssignError): + ctx.show_error( + f"Key {key} is not valid for {self_value}", + ErrorCode.incompatible_argument, + arg="key", + ) + return ImplReturn(KnownValue(None)) + + def _dict_pop_impl(ctx: CallContext) -> ImplReturn: key = ctx.vars["key"] default = ctx.vars["default"] @@ -780,13 +865,22 @@ def _dict_pop_impl(ctx: CallContext) -> ImplReturn: error_code=ErrorCode.incompatible_argument, arg="key", ) - if entry.readonly: + elif entry.readonly: ctx.show_error( f"Cannot pop readonly TypedDict key {key}", error_code=ErrorCode.readonly_typeddict, arg="key", ) return ImplReturn(_maybe_unite(entry.typ, default)) + if self_value.extra_keys_readonly: + ctx.show_error( + f"Cannot pop unknown key {key} in closed TypedDict {self_value}", + ErrorCode.readonly_typeddict, + arg="key", + ) + return ImplReturn( + _maybe_unite(self_value.extra_keys or TypedValue(object), default) + ) if ( self_value.extra_keys is not None and self_value.extra_keys is not NO_RETURN_VALUE @@ -1297,9 +1391,9 @@ def _str_format_impl(ctx: CallContext) -> Value: else: return TypedValue(str) elif isinstance(kwargs_value, TypedDictValue): - for key, (required, value_value) in kwargs_value.items.items(): - if required: - kwargs[key] = value_value + for key, entry in kwargs_value.items.items(): + if entry.required: + kwargs[key] = entry.typ else: return TypedValue(str) template = self.val @@ -1798,6 +1892,15 @@ def get_default_argspecs() -> Dict[object, Signature]: impl=_dict_pop_impl, return_annotation=AnyValue(AnySource.inference), ), + Signature.make( + [ + SigParameter("self", _POS_ONLY, annotation=TypedValue(dict)), + SigParameter("key", _POS_ONLY), + ], + callable=dict.__delitem__, + impl=_dict_delitem_impl, + return_annotation=KnownValue(None), + ), Signature.make( [ SigParameter("self", _POS_ONLY, annotation=TypedValue(dict)), diff --git a/pyanalyze/signature.py b/pyanalyze/signature.py index 1a48750c..e0ecbfbd 100644 --- a/pyanalyze/signature.py +++ b/pyanalyze/signature.py @@ -1038,7 +1038,8 @@ def bind_arguments( ) elif actual_args.star_kwargs is not None: value_value = unite_values( - *(val for _, val in items.values()), actual_args.star_kwargs + *(entry.typ for entry in items.values()), + actual_args.star_kwargs, ) star_kwargs_value = GenericValue( dict, [TypedValue(str), value_value] @@ -2132,7 +2133,9 @@ def _preprocess_kwargs_no_mvv( """ value = replace_known_sequence_value(value) if isinstance(value, TypedDictValue): - return value.items, None + return { + key: (entry.required, entry.typ) for key, entry in value.items.items() + }, None elif isinstance(value, DictIncompleteValue): return _preprocess_kwargs_kv_pairs(value.kv_pairs, ctx) else: @@ -2633,7 +2636,7 @@ def can_assign_var_keyword( return CanAssignError( f"parameter {my_param.name!r} is not accepted by {kwargs_annotation}" ) - their_annotation = kwargs_annotation.items[my_param.name][1] + their_annotation = kwargs_annotation.items[my_param.name].typ can_assign = their_annotation.can_assign(my_annotation, ctx) if isinstance(can_assign, CanAssignError): return CanAssignError( diff --git a/pyanalyze/test_typeddict.py b/pyanalyze/test_typeddict.py index 4d28e42e..f3899441 100644 --- a/pyanalyze/test_typeddict.py +++ b/pyanalyze/test_typeddict.py @@ -258,3 +258,105 @@ def caller() -> None: f({"a": "a"}) # E: incompatible_argument g({"c": 1.0}) g({}) # E: incompatible_argument + + +class TestReadOnly(TestNameCheckVisitorBase): + @assert_passes() + def test_basic(self): + from typing_extensions import NotRequired, ReadOnly, TypedDict + from typing import Any, Dict + + class TD(TypedDict): + a: ReadOnly[NotRequired[int]] + b: ReadOnly[str] + + def capybara(td: TD, anydict: Dict[str, Any]) -> None: + td["a"] = 1 # E: readonly_typeddict + td["b"] = "a" # E: readonly_typeddict + td.update(anydict) # E: invalid_typeddict_key + td.setdefault("a", 1) # E: readonly_typeddict + td.setdefault("b", "a") # E: readonly_typeddict + td.pop("a") # E: readonly_typeddict + td.pop("b") # E: incompatible_argument + del td["a"] # E: readonly_typeddict + del td["b"] # E: incompatible_argument + + @assert_passes() + def test_compatibility(self): + from typing_extensions import ReadOnly, TypedDict + from typing import Dict, Any + + class TD(TypedDict): + a: int + + class TD2(TypedDict): + a: ReadOnly[int] + + class TD3(TypedDict): + a: bool + + class TD4(TypedDict): + a: str + + def want_td(td: TD) -> None: + pass + + def want_td2(td: TD2) -> None: + pass + + def capybara( + td: TD, td2: TD2, td3: TD3, td4: TD4, anydict: Dict[str, Any] + ) -> None: + want_td(td) + want_td(td2) # E: incompatible_argument + want_td(td3) # E: incompatible_argument + want_td(td4) # E: incompatible_argument + want_td(anydict) + + want_td2(td) + want_td2(td2) + want_td2(td3) + want_td2(td4) # E: incompatible_argument + want_td2(anydict) + + +class TestClosed(TestNameCheckVisitorBase): + @assert_passes() + def test_basic(self): + from typing_extensions import NotRequired, TypedDict + from typing import Any, Dict + + class Closed(TypedDict, closed=True): + a: NotRequired[int] + b: str + + class Open(TypedDict): + a: NotRequired[int] + b: str + + def want_closed(td: Closed) -> None: + pass + + def want_open(td: Open) -> None: + pass + + def capybara(closed: Closed, open: Open, anydict: Dict[str, Any]) -> None: + closed["a"] = 1 + closed["b"] = "a" + closed["a"] = "x" # E: incompatible_argument + + open["a"] = 1 + open["b"] = "a" + open["a"] = "x" # E: incompatible_argument + + closed.update(anydict) # E: invalid_typeddict_key + open.update(anydict) # E: invalid_typeddict_key + + x: Closed = {"a": 1, "b": "a", "c": "x"} # E: incompatible_assignment + y: Open = {"a": 1, "b": "a", "c": "x"} + + want_closed(closed) + want_closed(open) # E: incompatible_argument + + want_open(open) + want_open(closed) diff --git a/pyanalyze/test_value.py b/pyanalyze/test_value.py index 5f132318..183c8cc0 100644 --- a/pyanalyze/test_value.py +++ b/pyanalyze/test_value.py @@ -358,7 +358,10 @@ def test_variable_name_value() -> None: def test_typeddict_value() -> None: val = value.TypedDictValue( - {"a": (True, TypedValue(int)), "b": (True, TypedValue(str))} + { + "a": value.TypedDictEntry(TypedValue(int)), + "b": value.TypedDictEntry(TypedValue(str)), + } ) # dict iteration order in some Python versions is not deterministic assert str(val) in [ @@ -384,26 +387,44 @@ def test_typeddict_value() -> None: assert_can_assign( val, value.TypedDictValue( - {"a": (True, KnownValue(1)), "b": (True, TypedValue(str))} + { + "a": value.TypedDictEntry(TypedValue(int)), + "b": value.TypedDictEntry(TypedValue(str)), + "c": value.TypedDictEntry(TypedValue(float)), + } ), ) - assert_can_assign( + assert_cannot_assign( val, value.TypedDictValue( { - "a": (True, KnownValue(1)), - "b": (True, TypedValue(str)), - "c": (True, TypedValue(float)), + "a": value.TypedDictEntry(KnownValue(1)), + "b": value.TypedDictEntry(TypedValue(str)), } ), ) assert_cannot_assign( val, value.TypedDictValue( - {"a": (True, KnownValue(1)), "b": (True, TypedValue(int))} + { + "a": value.TypedDictEntry(KnownValue(1)), + "b": value.TypedDictEntry(TypedValue(str)), + "c": value.TypedDictEntry(TypedValue(float)), + } ), ) - assert_cannot_assign(val, value.TypedDictValue({"b": (True, TypedValue(str))})) + assert_cannot_assign( + val, + value.TypedDictValue( + { + "a": value.TypedDictEntry(KnownValue(1)), + "b": value.TypedDictEntry(TypedValue(int)), + } + ), + ) + assert_cannot_assign( + val, value.TypedDictValue({"b": value.TypedDictEntry(TypedValue(str))}) + ) # DictIncompleteValue assert_can_assign( diff --git a/pyanalyze/value.py b/pyanalyze/value.py index 6454be99..260c4998 100644 --- a/pyanalyze/value.py +++ b/pyanalyze/value.py @@ -1387,6 +1387,10 @@ def can_assign(self, other: Value, ctx: CanAssignContext) -> CanAssign: return CanAssignError( f"Mutable key {key} is required in {other}" ) + if not entry.readonly and their_entry.readonly: + return CanAssignError( + f"Mutable key {key} is readonly in {other}" + ) can_assign = entry.typ.can_assign(their_entry.typ, ctx) if isinstance(can_assign, CanAssignError): @@ -1472,7 +1476,7 @@ def substitute_typevars(self, typevars: TypeVarMap) -> "TypedDictValue": ) def __str__(self) -> str: - entries = list(self.items.items()) + entries: List[Tuple[str, object]] = list(self.items.items()) if self.extra_keys is not None and self.extra_keys is not NO_RETURN_VALUE: extra_typ = str(self.extra_keys) if self.extra_keys_readonly: From d912bce04d30f1333c7c58ef3ed18181a792abb9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 23 Feb 2024 23:19:18 -0800 Subject: [PATCH 3/5] Fix remaining tests --- pyanalyze/test_annotations.py | 75 ++++++++++++++++++++--------------- pyanalyze/test_boolability.py | 5 ++- pyanalyze/test_signature.py | 22 ++++++---- pyanalyze/test_typeshed.py | 29 ++++++++++---- 4 files changed, 80 insertions(+), 51 deletions(-) diff --git a/pyanalyze/test_annotations.py b/pyanalyze/test_annotations.py index 657e4ecf..035ae38a 100644 --- a/pyanalyze/test_annotations.py +++ b/pyanalyze/test_annotations.py @@ -15,6 +15,7 @@ NewTypeValue, SequenceValue, SubclassValue, + TypedDictEntry, TypedDictValue, TypedValue, TypeVarValue, @@ -408,7 +409,8 @@ def capybara( @skip_before((3, 9)) def test_builtin_tuples_string(self): - self.assert_passes(""" + self.assert_passes( + """ from __future__ import annotations from collections.abc import Iterable from typing import Union @@ -434,7 +436,8 @@ def capybara( assert_is_value(t, t_str_int) for elt in returner(): assert_is_value(elt, t_str_int) - """) + """ + ) @assert_passes() def test_invalid_annotation(self): @@ -490,14 +493,16 @@ def capybara(x: Pattern[str]): assert_is_value(x, GenericValue(_Pattern, [TypedValue(str)])) def test_future_annotations(self): - self.assert_passes(""" + self.assert_passes( + """ from __future__ import annotations from typing import List def f(x: int, y: List[str]): assert_is_value(x, TypedValue(int)) assert_is_value(y, GenericValue(list, [TypedValue(str)])) - """) + """ + ) @assert_passes() def test_final(self): @@ -556,7 +561,8 @@ def capybara(x: list[int], y: tuple[int, str], z: tuple[int, ...]) -> None: @skip_before((3, 9)) def test_pep604(self): - self.assert_passes(""" + self.assert_passes( + """ from __future__ import annotations def capybara(x: int | None, y: int | str) -> None: @@ -566,11 +572,13 @@ def capybara(x: int | None, y: int | str) -> None: def caller(): capybara(1, 2) capybara(None, "x") - """) + """ + ) @skip_before((3, 10)) def test_pep604_runtime(self): - self.assert_passes(""" + self.assert_passes( + """ def capybara(x: int | None, y: int | str) -> None: assert_is_value(x, MultiValuedValue([TypedValue(int), KnownValue(None)])) assert_is_value(y, MultiValuedValue([TypedValue(int), TypedValue(str)])) @@ -578,7 +586,8 @@ def capybara(x: int | None, y: int | str) -> None: def caller(): capybara(1, 2) capybara(None, "x") - """) + """ + ) @assert_passes() def test_stringified_ops(self): @@ -1300,7 +1309,7 @@ def deep(x: Annotated[List[int], NoAny(deep=True)]) -> None: pass def none_at_all( - x: Annotated[List[int], NoAny(deep=True, allowed_sources=frozenset())] + x: Annotated[List[int], NoAny(deep=True, allowed_sources=frozenset())], ) -> None: pass @@ -1496,9 +1505,9 @@ def take_rnr(td: RNR) -> None: td, TypedDictValue( { - "a": (True, TypedValue(int)), - "b": (True, TypedValue(str)), - "c": (False, TypedValue(float)), + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(str)), + "c": TypedDictEntry(TypedValue(float), required=False), } ), ) @@ -1513,9 +1522,9 @@ def take_not_total(td: NotTotal) -> None: td, TypedDictValue( { - "a": (False, TypedValue(int)), - "b": (True, TypedValue(str)), - "c": (False, TypedValue(float)), + "a": TypedDictEntry(TypedValue(int), required=False), + "b": TypedDictEntry(TypedValue(str)), + "c": TypedDictEntry(TypedValue(float), required=False), } ), ) @@ -1530,9 +1539,9 @@ def take_stringify(td: Stringify) -> None: td, TypedDictValue( { - "a": (True, TypedValue(int)), - "b": (True, TypedValue(str)), - "c": (False, TypedValue(float)), + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(str)), + "c": TypedDictEntry(TypedValue(float), required=False), } ), ) @@ -1563,9 +1572,9 @@ def capybara() -> None: KnownValue(None) | TypedDictValue( { - "a": (True, TypedValue(int)), - "b": (True, TypedValue(str)), - "c": (False, TypedValue(float)), + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(str)), + "c": TypedDictEntry(TypedValue(float), required=False), } ), ) @@ -1574,9 +1583,9 @@ def capybara() -> None: KnownValue(None) | TypedDictValue( { - "a": (True, TypedValue(int)), - "b": (True, TypedValue(str)), - "c": (False, TypedValue(float)), + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(str)), + "c": TypedDictEntry(TypedValue(float), required=False), } ), ) @@ -1597,9 +1606,9 @@ def take_rnr(td: RNR) -> None: td, TypedDictValue( { - "a": (True, TypedValue(int)), - "b": (True, TypedValue(str)), - "c": (False, TypedValue(float)), + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(str)), + "c": TypedDictEntry(TypedValue(float), required=False), } ), ) @@ -1614,9 +1623,9 @@ def take_not_total(td: NotTotal) -> None: td, TypedDictValue( { - "a": (False, TypedValue(int)), - "b": (True, TypedValue(str)), - "c": (False, TypedValue(float)), + "a": TypedDictEntry(TypedValue(int), required=False), + "b": TypedDictEntry(TypedValue(str)), + "c": TypedDictEntry(TypedValue(float), required=False), } ), ) @@ -1631,9 +1640,9 @@ def take_stringify(td: Stringify) -> None: td, TypedDictValue( { - "a": (True, TypedValue(int)), - "b": (True, TypedValue(str)), - "c": (False, TypedValue(float)), + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(str)), + "c": TypedDictEntry(TypedValue(float), required=False), } ), ) diff --git a/pyanalyze/test_boolability.py b/pyanalyze/test_boolability.py index 446afa90..1bf88b16 100644 --- a/pyanalyze/test_boolability.py +++ b/pyanalyze/test_boolability.py @@ -14,6 +14,7 @@ KVPair, NO_RETURN_VALUE, SequenceValue, + TypedDictEntry, TypedDictValue, TypedValue, UnboundMethodValue, @@ -44,10 +45,10 @@ def test_get_boolability() -> None: # Sequence/dict values assert Boolability.type_always_true == get_boolability( - TypedDictValue({"a": (True, TypedValue(int))}) + TypedDictValue({"a": TypedDictEntry(TypedValue(int))}) ) assert Boolability.boolable == get_boolability( - TypedDictValue({"a": (False, TypedValue(int))}) + TypedDictValue({"a": TypedDictEntry(TypedValue(int), required=False)}) ) assert Boolability.type_always_true == get_boolability( SequenceValue(tuple, [(False, KnownValue(1))]) diff --git a/pyanalyze/test_signature.py b/pyanalyze/test_signature.py index 17824d9e..c41f7ac0 100644 --- a/pyanalyze/test_signature.py +++ b/pyanalyze/test_signature.py @@ -23,6 +23,7 @@ GenericValue, KnownValue, SequenceValue, + TypedDictEntry, TypedDictValue, TypedValue, ) @@ -322,9 +323,9 @@ def test_advanced_var_keyword(self) -> None: self.can(three_ints_sig, Signature.make([dict_int])) good_td = TypedDictValue( { - "a": (True, TypedValue(int)), - "b": (True, TypedValue(int)), - "c": (True, TypedValue(int)), + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(int)), + "c": TypedDictEntry(TypedValue(int)), } ) self.can( @@ -332,7 +333,7 @@ def test_advanced_var_keyword(self) -> None: Signature.make([P("a", annotation=good_td, kind=K.VAR_KEYWORD)]), ) smaller_td = TypedDictValue( - {"a": (True, TypedValue(int)), "b": (True, TypedValue(int))} + {"a": TypedDictEntry(TypedValue(int)), "b": TypedDictEntry(TypedValue(int))} ) # This is still OK because TypedDicts are allowed to have extra keys. # TODO change to can @@ -341,7 +342,7 @@ def test_advanced_var_keyword(self) -> None: Signature.make([P("a", annotation=smaller_td, kind=K.VAR_KEYWORD)]), ) bad_td = TypedDictValue( - {"a": (True, TypedValue(str)), "b": (True, TypedValue(int))} + {"a": TypedDictEntry(TypedValue(str)), "b": TypedDictEntry(TypedValue(int))} ) self.cannot( three_ints_sig, @@ -862,7 +863,8 @@ def capybara(arg): many_args(**typed_int_kwargs) # E: incompatible_call def test_pos_only(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import Sequence def pos_only(pos: int, /) -> None: @@ -875,7 +877,8 @@ def capybara(ints: Sequence[int], strs: Sequence[str]) -> None: pos_only(pos=1) # E: incompatible_call pos_only() # E: incompatible_call pos_only(1, 2) # E: incompatible_call - """) + """ + ) @assert_passes() def test_kw_only(self): @@ -1280,7 +1283,10 @@ def capybara(**kwargs: Unpack[TD]): assert_is_value( kwargs, TypedDictValue( - {"a": (False, TypedValue(int)), "b": (True, TypedValue(str))} + { + "a": TypedDictEntry(TypedValue(int), required=False), + "b": TypedDictEntry(TypedValue(str)), + } ), ) diff --git a/pyanalyze/test_typeshed.py b/pyanalyze/test_typeshed.py index dc78f3fa..3c40c61e 100644 --- a/pyanalyze/test_typeshed.py +++ b/pyanalyze/test_typeshed.py @@ -30,6 +30,7 @@ AnySource, AnyValue, SequenceValue, + TypedDictEntry, assert_is_value, CallableValue, DictIncompleteValue, @@ -84,7 +85,9 @@ def test_newtype(self): with tempfile.TemporaryDirectory() as temp_dir_str: temp_dir = Path(temp_dir_str) (temp_dir / "typing.pyi").write_text("def NewType(a, b): pass\n") - (temp_dir / "newt.pyi").write_text(textwrap.dedent(""" + (temp_dir / "newt.pyi").write_text( + textwrap.dedent( + """ from typing import NewType NT = NewType("NT", int) @@ -92,7 +95,9 @@ def test_newtype(self): def f(x: NT, y: Alias) -> None: pass - """)) + """ + ) + ) (temp_dir / "VERSIONS").write_text("newt: 3.5\ntyping: 3.5\n") (temp_dir / "@python2").mkdir() tsf = TypeshedFinder(Checker(), verbose=True) @@ -176,18 +181,26 @@ def test_get_attribute(self) -> None: _EXPECTED_TYPED_DICTS = { - "TD1": TypedDictValue({"a": (True, TypedValue(int)), "b": (True, TypedValue(str))}), + "TD1": TypedDictValue( + {"a": TypedDictEntry(TypedValue(int)), "b": TypedDictEntry(TypedValue(str))} + ), "TD2": TypedDictValue( - {"a": (False, TypedValue(int)), "b": (False, TypedValue(str))} + { + "a": TypedDictEntry(TypedValue(int), required=False), + "b": TypedDictEntry(TypedValue(str), required=False), + } ), "PEP655": TypedDictValue( - {"a": (False, TypedValue(int)), "b": (True, TypedValue(str))} + { + "a": TypedDictEntry(TypedValue(int), required=False), + "b": TypedDictEntry(TypedValue(str)), + } ), "Inherited": TypedDictValue( { - "a": (True, TypedValue(int)), - "b": (True, TypedValue(str)), - "c": (True, TypedValue(float)), + "a": TypedDictEntry(TypedValue(int)), + "b": TypedDictEntry(TypedValue(str)), + "c": TypedDictEntry(TypedValue(float)), } ), } From 4e168808e3ff4f589351c2cfc197e0e3984acf90 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 23 Feb 2024 23:21:44 -0800 Subject: [PATCH 4/5] more formatting --- pyanalyze/test_ast_annotator.py | 18 +++++++---- pyanalyze/test_functions.py | 12 +++++--- pyanalyze/test_import.py | 6 ++-- pyanalyze/test_node_visitor.py | 6 +++- pyanalyze/test_patma.py | 54 ++++++++++++++++++++++----------- pyanalyze/test_try.py | 24 ++++++++++----- pyanalyze/test_type_aliases.py | 18 +++++++---- pyanalyze/test_typevar.py | 12 +++++--- pyanalyze/value.py | 12 +++++--- 9 files changed, 109 insertions(+), 53 deletions(-) diff --git a/pyanalyze/test_ast_annotator.py b/pyanalyze/test_ast_annotator.py index 6fff4645..faee0513 100644 --- a/pyanalyze/test_ast_annotator.py +++ b/pyanalyze/test_ast_annotator.py @@ -25,23 +25,28 @@ def test_annotate_code() -> None: _check_inferred_value(tree, ast.Constant, KnownValue(1)) _check_inferred_value(tree, ast.Name, KnownValue(1)) - tree = annotate_code(""" + tree = annotate_code( + """ class X: def __init__(self): self.a = 1 - """) + """ + ) _check_inferred_value(tree, ast.Attribute, KnownValue(1)) - tree = annotate_code(""" + tree = annotate_code( + """ class X: def __init__(self): self.a = 1 x = X() x.a + 1 - """) + """ + ) _check_inferred_value(tree, ast.BinOp, KnownValue(2)) - tree = annotate_code(""" + tree = annotate_code( + """ class A: def __init__(self): self.a = 1 @@ -52,7 +57,8 @@ def bla(self): a = A() b = a.bla() - """) + """ + ) _check_inferred_value(tree, ast.Name, KnownValue(1), lambda node: node.id == "b") diff --git a/pyanalyze/test_functions.py b/pyanalyze/test_functions.py index e0563754..d02c3bde 100644 --- a/pyanalyze/test_functions.py +++ b/pyanalyze/test_functions.py @@ -219,7 +219,8 @@ def caller() -> None: class TestGenericFunctions(TestNameCheckVisitorBase): @skip_before((3, 12)) def test_generic(self): - self.assert_passes(""" + self.assert_passes( + """ from typing_extensions import assert_type def func[T](x: T) -> T: @@ -227,11 +228,13 @@ def func[T](x: T) -> T: def capybara(i: int): assert_type(func(i), int) - """) + """ + ) @skip_before((3, 12)) def test_generic_with_bound(self): - self.assert_passes(""" + self.assert_passes( + """ from typing_extensions import assert_type def func[T: int](x: T) -> T: @@ -241,4 +244,5 @@ def capybara(i: int, s: str, b: bool): assert_type(func(i), int) assert_type(func(b), bool) func(s) # E: incompatible_argument - """) + """ + ) diff --git a/pyanalyze/test_import.py b/pyanalyze/test_import.py index 19be5642..f4a3a347 100644 --- a/pyanalyze/test_import.py +++ b/pyanalyze/test_import.py @@ -32,7 +32,8 @@ def capybara(): assert_is_value(assert_error, KnownValue(P.extensions.assert_error)) def test_import_star(self): - self.assert_passes(""" + self.assert_passes( + """ import pyanalyze as P if False: @@ -40,7 +41,8 @@ def test_import_star(self): assert_is_value(extensions, KnownValue(P.extensions)) not_a_name # E: undefined_name - """) + """ + ) class TestDisallowedImport(TestNameCheckVisitorBase): diff --git a/pyanalyze/test_node_visitor.py b/pyanalyze/test_node_visitor.py index f3187fc8..9c607a69 100644 --- a/pyanalyze/test_node_visitor.py +++ b/pyanalyze/test_node_visitor.py @@ -407,7 +407,11 @@ def assert_code_equal(expected, actual): %s >>> diff: %s -""" % (expected, actual, diff) +""" % ( + expected, + actual, + diff, + ) assert False, message diff --git a/pyanalyze/test_patma.py b/pyanalyze/test_patma.py index eb31edeb..e8d3b1f9 100644 --- a/pyanalyze/test_patma.py +++ b/pyanalyze/test_patma.py @@ -6,7 +6,8 @@ class TestPatma(TestNameCheckVisitorBase): @skip_before((3, 10)) def test_singletons(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import Literal def capybara(x: Literal[True, False, None]): match x: @@ -14,11 +15,13 @@ def capybara(x: Literal[True, False, None]): assert_is_value(x, KnownValue(True)) case _: assert_is_value(x, KnownValue(False) | KnownValue(None)) - """) + """ + ) @skip_before((3, 10)) def test_value(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import Literal from pyanalyze.tests import assert_never @@ -35,11 +38,13 @@ def capybara(x: int): assert_is_value(x, KnownValue(4)) case _: assert_is_value(x, TypedValue(int)) - """) + """ + ) @skip_before((3, 10)) def test_sequence(self): - self.assert_passes(""" + self.assert_passes( + """ import collections.abc from typing import Tuple @@ -72,11 +77,13 @@ def capybara(seq: Tuple[int, ...], obj: object): match seq[0]: case [1, 2, 3]: # E: impossible_pattern pass - """) + """ + ) @skip_before((3, 10)) def test_or(self): - self.assert_passes(""" + self.assert_passes( + """ import collections.abc from typing import Tuple @@ -86,11 +93,13 @@ def capybara(obj: object): assert_is_value(obj, KnownValue(1) | KnownValue(2)) case (3 as x) | (4 as x): assert_is_value(x, KnownValue(3) | KnownValue(4)) - """) + """ + ) @skip_before((3, 10)) def test_mapping(self): - self.assert_passes(""" + self.assert_passes( + """ import collections.abc from typing import Tuple @@ -105,11 +114,13 @@ def capybara(obj: object): KVPair(KnownValue(5), KnownValue(6)), ] )) - """) + """ + ) @skip_before((3, 10)) def test_class_pattern(self): - self.assert_passes(""" + self.assert_passes( + """ import collections.abc from typing import Tuple @@ -147,11 +158,13 @@ def capybara(obj: object): pass case MatchArgs(1, 2, 3): # E: bad_match pass - """) + """ + ) @skip_before((3, 10)) def test_bool_narrowing(self): - self.assert_passes(""" + self.assert_passes( + """ class X: true = True @@ -163,8 +176,10 @@ def capybara(b: bool): case _ as b2: assert_is_value(b, KnownValue(False)) assert_is_value(b2, KnownValue(False)) - """) - self.assert_passes(""" + """ + ) + self.assert_passes( + """ def capybara(b: bool): match b: case True: @@ -172,11 +187,13 @@ def capybara(b: bool): case _ as b2: assert_is_value(b, KnownValue(False)) assert_is_value(b2, KnownValue(False)) - """) + """ + ) @skip_before((3, 10)) def test_enum_narrowing(self): - self.assert_passes(""" + self.assert_passes( + """ from enum import Enum class Planet(Enum): @@ -193,4 +210,5 @@ def capybara(p: Planet): case _ as p2: assert_is_value(p2, KnownValue(Planet.earth)) assert_is_value(p, KnownValue(Planet.earth)) - """) + """ + ) diff --git a/pyanalyze/test_try.py b/pyanalyze/test_try.py index 4d71ed9a..dd3e0888 100644 --- a/pyanalyze/test_try.py +++ b/pyanalyze/test_try.py @@ -37,7 +37,8 @@ def capybara( class TestTryStar(TestNameCheckVisitorBase): @skip_before((3, 11)) def test_eg_types(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import assert_type def capybara(): @@ -53,11 +54,13 @@ def capybara(): pass except *int as eg: # E: bad_except_handler pass - """) + """ + ) @skip_before((3, 11)) def test_variable_scope(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import assert_type, Literal def capybara(): @@ -72,11 +75,13 @@ def capybara(): assert_type(x, Literal[0, 1, 2]) x = 3 assert_type(x, Literal[1, 2, 3]) - """) + """ + ) @skip_before((3, 11)) def test_try_else(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import assert_type, Literal def capybara(): @@ -94,11 +99,13 @@ def capybara(): assert_type(x, Literal[1]) x = 4 assert_type(x, Literal[2, 3, 4]) - """) + """ + ) @skip_before((3, 11)) def test_try_finally(self): - self.assert_passes(""" + self.assert_passes( + """ from typing import assert_type, Literal def capybara(): @@ -116,4 +123,5 @@ def capybara(): assert_type(x, Literal[0, 1, 2, 3]) x = 4 assert_type(x, Literal[4]) - """) + """ + ) diff --git a/pyanalyze/test_type_aliases.py b/pyanalyze/test_type_aliases.py index 0411f7a5..a1c81366 100644 --- a/pyanalyze/test_type_aliases.py +++ b/pyanalyze/test_type_aliases.py @@ -52,7 +52,8 @@ def capybara(i: int, s: str): @skip_before((3, 12)) def test_312(self): - self.assert_passes(""" + self.assert_passes( + """ from typing_extensions import assert_type type MyType = int @@ -63,11 +64,13 @@ def f(x: MyType): def capybara(i: int, s: str): f(i) f(s) # E: incompatible_argument - """) + """ + ) @skip_before((3, 12)) def test_312_generic(self): - self.assert_passes(""" + self.assert_passes( + """ from typing_extensions import assert_type type MyType[T] = list[T] | set[T] @@ -78,11 +81,13 @@ def f(x: MyType[int]): def capybara(i: int, s: str): f([i]) f([s]) # E: incompatible_argument - """) + """ + ) @skip_before((3, 12)) def test_312_local_alias(self): - self.assert_passes(""" + self.assert_passes( + """ from typing_extensions import assert_type def capybara(): @@ -93,4 +98,5 @@ def f(x: MyType): f(1) f("x") # E: incompatible_argument - """) + """ + ) diff --git a/pyanalyze/test_typevar.py b/pyanalyze/test_typevar.py index 84d11264..fcc2a781 100644 --- a/pyanalyze/test_typevar.py +++ b/pyanalyze/test_typevar.py @@ -389,7 +389,8 @@ def capybara(s: Sequence[int], t: str): class TestGenericClasses(TestNameCheckVisitorBase): @skip_before((3, 12)) def test_generic(self): - self.assert_passes(""" + self.assert_passes( + """ from typing_extensions import assert_type class C[T]: @@ -400,11 +401,13 @@ def __init__(self, x: T) -> None: def capybara(i: int): assert_type(C(i).x, int) - """) + """ + ) @skip_before((3, 12)) def test_generic_with_bound(self): - self.assert_passes(""" + self.assert_passes( + """ from typing_extensions import assert_type class C[T: int]: @@ -417,4 +420,5 @@ def capybara(i: int, s: str, b: bool): assert_type(C(i).x, int) assert_type(C(b).x, bool) C(s) # E: incompatible_argument - """) + """ + ) diff --git a/pyanalyze/value.py b/pyanalyze/value.py index 260c4998..64ef605d 100644 --- a/pyanalyze/value.py +++ b/pyanalyze/value.py @@ -1321,13 +1321,15 @@ def can_assign(self, other: Value, ctx: CanAssignContext) -> CanAssign: if key_type.val not in self.items: if self.extra_keys is NO_RETURN_VALUE: return CanAssignError( - f"Key {key_type.val!r} is not allowed in closed TypedDict {self}" + f"Key {key_type.val!r} is not allowed in closed" + f" TypedDict {self}" ) elif self.extra_keys is not None: can_assign = self.extra_keys.can_assign(pair.value, ctx) if isinstance(can_assign, CanAssignError): return CanAssignError( - f"Type for extra key {pair.key} is incompatible", + f"Type for extra key {pair.key} is" + " incompatible", children=[can_assign], ) bounds_maps.append(can_assign) @@ -1340,7 +1342,8 @@ def can_assign(self, other: Value, ctx: CanAssignContext) -> CanAssign: ) if self.extra_keys is NO_RETURN_VALUE: return CanAssignError( - f"Key {pair.key} is not allowed in closed TypedDict {self}" + f"Key {pair.key} is not allowed in closed TypedDict" + f" {self}" ) elif self.extra_keys is not None: can_assign = self.extra_keys.can_assign(pair.value, ctx) @@ -1368,7 +1371,8 @@ def can_assign(self, other: Value, ctx: CanAssignContext) -> CanAssign: can_assign = entry.typ.can_assign(extra_keys_type, ctx) if isinstance(can_assign, CanAssignError): return CanAssignError( - f"Type for key {key} is incompatible with extra keys type {extra_keys_type}", + f"Type for key {key} is incompatible with extra keys type" + f" {extra_keys_type}", children=[can_assign], ) else: From bcdd830cba191f2cefef0091d83c319ea47379de Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 24 Feb 2024 00:11:24 -0800 Subject: [PATCH 5/5] reformat --- pyanalyze/annotations.py | 32 ++++++++++++++--------------- pyanalyze/signature.py | 40 ++++++++++++++++++------------------- pyanalyze/test_typeddict.py | 35 +++++++++++++++++++++----------- pyanalyze/test_typeshed.py | 23 +++++++++++---------- pyanalyze/typeshed.py | 33 ++++++++++++++---------------- 5 files changed, 85 insertions(+), 78 deletions(-) diff --git a/pyanalyze/annotations.py b/pyanalyze/annotations.py index 7531c407..2c65fa15 100644 --- a/pyanalyze/annotations.py +++ b/pyanalyze/annotations.py @@ -29,10 +29,10 @@ import contextlib import typing from collections.abc import Callable, Hashable -from dataclasses import dataclass, field, InitVar +from dataclasses import InitVar, dataclass, field from typing import ( + TYPE_CHECKING, Any, - cast, Container, ContextManager, Generator, @@ -43,20 +43,18 @@ Sequence, Set, Tuple, - TYPE_CHECKING, TypeVar, Union, + cast, ) -import typing_extensions import qcore - -from typing_extensions import Literal, ParamSpec, TypedDict, get_origin, get_args +import typing_extensions +from typing_extensions import Literal, ParamSpec, TypedDict, get_args, get_origin from pyanalyze.annotated_types import get_annotated_types_extension from . import type_evaluation - from .error_code import ErrorCode from .extensions import ( AsynqCallable, @@ -81,26 +79,20 @@ SigParameter, ) from .value import ( - _HashableValue, - DictIncompleteValue, - KVPair, - TypeAlias, - TypeAliasValue, - TypeIsExtension, - TypedDictEntry, - annotate_value, + NO_RETURN_VALUE, AnnotatedValue, AnySource, AnyValue, CallableValue, CustomCheckExtension, + DictIncompleteValue, Extension, GenericValue, HasAttrGuardExtension, KnownValue, + KVPair, MultiValuedValue, NewTypeValue, - NO_RETURN_VALUE, NoReturnGuardExtension, ParameterTypeGuardExtension, ParamSpecArgsValue, @@ -108,14 +100,20 @@ SelfTVV, SequenceValue, SubclassValue, + TypeAlias, + TypeAliasValue, + TypedDictEntry, TypedDictValue, TypedValue, TypeGuardExtension, + TypeIsExtension, TypeVarLike, TypeVarValue, - unite_values, UnpackedValue, Value, + _HashableValue, + annotate_value, + unite_values, ) if TYPE_CHECKING: diff --git a/pyanalyze/signature.py b/pyanalyze/signature.py index e0ecbfbd..ffa30f6d 100644 --- a/pyanalyze/signature.py +++ b/pyanalyze/signature.py @@ -14,6 +14,7 @@ from dataclasses import dataclass, field, replace from types import FunctionType, MethodType from typing import ( + TYPE_CHECKING, Any, Callable, ClassVar, @@ -26,7 +27,6 @@ Sequence, Set, Tuple, - TYPE_CHECKING, TypeVar, Union, ) @@ -34,83 +34,83 @@ import asynq import qcore from qcore.helpers import safe_str -from typing_extensions import assert_never, Literal, Protocol, Self +from typing_extensions import Literal, Protocol, Self, assert_never from pyanalyze.predicates import IsAssignablePredicate from .error_code import ErrorCode -from .safe import safe_getattr from .node_visitor import Replacement from .options import IntegerOption +from .safe import safe_getattr from .stacked_scopes import ( + NULL_CONSTRAINT, AbstractConstraint, AndConstraint, Composite, Constraint, ConstraintType, - NULL_CONSTRAINT, OrConstraint, VarnameWithOrigin, ) from .type_evaluation import ( ARGS, DEFAULT, + KWARGS, + UNKNOWN, EvalContext, Evaluator, - KWARGS, Position, - UNKNOWN, ) from .typevar import resolve_bounds_map from .value import ( - SelfT, - TypeIsExtension, - TypedDictEntry, - annotate_value, + NO_RETURN_VALUE, AnnotatedValue, AnySource, AnyValue, AsyncTaskIncompleteValue, BoundsMap, CallableValue, - can_assign_and_used_any, CanAssign, CanAssignContext, CanAssignError, - concrete_values_from_iterable, ConstraintExtension, DictIncompleteValue, - extract_typevars, - flatten_values, GenericValue, - get_tv_map, HasAttrExtension, HasAttrGuardExtension, KnownValue, KVPair, LowerBound, MultiValuedValue, - NO_RETURN_VALUE, NoReturnConstraintExtension, NoReturnGuardExtension, ParameterTypeGuardExtension, ParamSpecArgsValue, ParamSpecKwargsValue, - is_iterable, - replace_known_sequence_value, + SelfT, SequenceValue, - stringify_object, + TypedDictEntry, TypedDictValue, TypedValue, TypeGuardExtension, + TypeIsExtension, TypeVarLike, TypeVarMap, TypeVarValue, + Value, + annotate_value, + can_assign_and_used_any, + concrete_values_from_iterable, + extract_typevars, + flatten_values, + get_tv_map, + is_iterable, + replace_known_sequence_value, + stringify_object, unannotate, unannotate_value, unify_bounds_maps, unite_values, - Value, ) if TYPE_CHECKING: diff --git a/pyanalyze/test_typeddict.py b/pyanalyze/test_typeddict.py index f3899441..441e6d6f 100644 --- a/pyanalyze/test_typeddict.py +++ b/pyanalyze/test_typeddict.py @@ -2,15 +2,16 @@ from .implementation import assert_is_value from .test_name_check_visitor import TestNameCheckVisitorBase from .test_node_visitor import assert_passes -from .value import TypedDictValue, TypedValue, AnyValue, AnySource, TypedDictEntry +from .value import AnySource, AnyValue, TypedDictEntry, TypedDictValue, TypedValue class TestExtraKeys(TestNameCheckVisitorBase): @assert_passes() def test_signature(self): - from pyanalyze.extensions import has_extra_keys from typing_extensions import TypedDict + from pyanalyze.extensions import has_extra_keys + @has_extra_keys(int) class TD(TypedDict): a: str @@ -28,10 +29,12 @@ def capybara() -> None: @assert_passes() def test_methods(self): - from pyanalyze.extensions import has_extra_keys - from typing_extensions import TypedDict, assert_type, Literal from typing import Union + from typing_extensions import Literal, TypedDict, assert_type + + from pyanalyze.extensions import has_extra_keys + @has_extra_keys(int) class TD(TypedDict): a: str @@ -57,9 +60,10 @@ def setdefault(td: TD) -> None: @assert_passes() def test_kwargs_annotation(self): - from pyanalyze.extensions import has_extra_keys from typing_extensions import TypedDict, Unpack, assert_type + from pyanalyze.extensions import has_extra_keys + @has_extra_keys(int) class TD(TypedDict): a: str @@ -73,10 +77,12 @@ def capybara() -> None: @assert_passes() def test_compatibility(self): - from pyanalyze.extensions import has_extra_keys - from typing_extensions import ReadOnly, TypedDict from typing import Any, Dict + from typing_extensions import ReadOnly, TypedDict + + from pyanalyze.extensions import has_extra_keys + @has_extra_keys(int) class TD(TypedDict): a: str @@ -112,10 +118,12 @@ def capybara(td: TD, td2: TD2, td3: TD3, anydict: Dict[str, Any]) -> None: @assert_passes() def test_iteration(self): - from pyanalyze.extensions import has_extra_keys - from typing_extensions import TypedDict, assert_type, Literal from typing import Union + from typing_extensions import Literal, TypedDict, assert_type + + from pyanalyze.extensions import has_extra_keys + @has_extra_keys(int) class TD(TypedDict): a: str @@ -263,9 +271,10 @@ def caller() -> None: class TestReadOnly(TestNameCheckVisitorBase): @assert_passes() def test_basic(self): - from typing_extensions import NotRequired, ReadOnly, TypedDict from typing import Any, Dict + from typing_extensions import NotRequired, ReadOnly, TypedDict + class TD(TypedDict): a: ReadOnly[NotRequired[int]] b: ReadOnly[str] @@ -283,8 +292,9 @@ def capybara(td: TD, anydict: Dict[str, Any]) -> None: @assert_passes() def test_compatibility(self): + from typing import Any, Dict + from typing_extensions import ReadOnly, TypedDict - from typing import Dict, Any class TD(TypedDict): a: int @@ -323,9 +333,10 @@ def capybara( class TestClosed(TestNameCheckVisitorBase): @assert_passes() def test_basic(self): - from typing_extensions import NotRequired, TypedDict from typing import Any, Dict + from typing_extensions import NotRequired, TypedDict + class Closed(TypedDict, closed=True): a: NotRequired[int] b: str diff --git a/pyanalyze/test_typeshed.py b/pyanalyze/test_typeshed.py index 3c40c61e..41e077aa 100644 --- a/pyanalyze/test_typeshed.py +++ b/pyanalyze/test_typeshed.py @@ -12,10 +12,10 @@ from collections.abc import Collection, MutableSequence, Reversible, Sequence, Set from pathlib import Path from typing import Dict, Generic, List, NewType, TypeVar, Union +from unittest.mock import ANY from urllib.error import HTTPError -from unittest.mock import ANY -from typeshed_client import get_search_context, Resolver +from typeshed_client import Resolver, get_search_context from .checker import Checker from .extensions import evaluated @@ -27,23 +27,23 @@ from .tests import make_simple_sequence from .typeshed import TypeshedFinder from .value import ( + UNINITIALIZED_VALUE, AnySource, AnyValue, - SequenceValue, - TypedDictEntry, - assert_is_value, CallableValue, DictIncompleteValue, GenericValue, KnownValue, KVPair, NewTypeValue, + SequenceValue, SubclassValue, + TypedDictEntry, TypedDictValue, TypedValue, TypeVarValue, - UNINITIALIZED_VALUE, Value, + assert_is_value, ) T = TypeVar("T") @@ -161,6 +161,7 @@ def capybara(i: int) -> None: @assert_passes() def test_datetime(self): from datetime import datetime + from typing_extensions import assert_type def capybara(i: int): @@ -211,9 +212,9 @@ class TestBundledStubs(TestNameCheckVisitorBase): def test_import_aliases(self): def capybara(): from _pyanalyze_tests.aliases import ( + ExplicitAlias, aliased_constant, constant, - ExplicitAlias, explicitly_aliased_constant, ) @@ -284,7 +285,7 @@ def capybara(x: ast.Yield): @assert_passes() def test_import_typeddicts(self): def capybara(): - from _pyanalyze_tests.typeddict import Inherited, PEP655, TD1, TD2 + from _pyanalyze_tests.typeddict import PEP655, TD1, TD2, Inherited from pyanalyze.test_typeshed import _EXPECTED_TYPED_DICTS @@ -302,7 +303,7 @@ def test_evaluated(self): @assert_passes() def test_evaluated_import(self): def capybara(unannotated): - from typing import BinaryIO, IO, TextIO + from typing import IO, BinaryIO, TextIO from _pyanalyze_tests.evaluated import open, open2 @@ -357,7 +358,7 @@ def capybara(): @assert_passes() def test_stub_context_manager(self): - from typing_extensions import assert_type, Literal + from typing_extensions import Literal, assert_type def capybara(): from _pyanalyze_tests.contextmanager import cm @@ -760,7 +761,7 @@ class TestIntegration(TestNameCheckVisitorBase): @assert_passes() def test_open(self): import io - from typing import Any, BinaryIO, IO + from typing import IO, Any, BinaryIO from pyanalyze.extensions import assert_type diff --git a/pyanalyze/typeshed.py b/pyanalyze/typeshed.py index 1a4a92fa..9dc1dacd 100644 --- a/pyanalyze/typeshed.py +++ b/pyanalyze/typeshed.py @@ -10,13 +10,13 @@ import enum import inspect import sys - +import types from abc import abstractmethod -from collections.abc import Collection, MutableMapping, Set as AbstractSet +from collections.abc import Collection, MutableMapping +from collections.abc import Set as AbstractSet from dataclasses import dataclass, field, replace from enum import Enum, EnumMeta from types import GeneratorType, MethodDescriptorType, ModuleType -import types from typing import ( Any, Callable, @@ -37,55 +37,52 @@ from typing_extensions import Protocol, TypedDict from pyanalyze.functions import translate_vararg_type + from .analysis_lib import is_positional_only_arg_name from .annotations import ( Context, - TypeQualifierValue, - make_type_var_value, DecoratorValue, SyntheticEvaluator, + TypeQualifierValue, + make_type_var_value, type_from_value, value_from_ast, ) from .error_code import ErrorCode -from .extensions import ( - evaluated, - overload, - real_overload, - deprecated as deprecated_decorator, -) +from .extensions import deprecated as deprecated_decorator +from .extensions import evaluated, overload, real_overload from .node_visitor import Failure from .options import Options, PathSequenceOption from .safe import all_of_type, hasattr_static, is_typing_name, safe_isinstance from .signature import ( ConcreteSignature, - make_bound_method, OverloadedSignature, ParameterKind, Signature, SigParameter, + make_bound_method, ) from .stacked_scopes import Composite, uniq_chain from .value import ( + UNINITIALIZED_VALUE, AnySource, AnyValue, CallableValue, CanAssignContext, DeprecatedExtension, Extension, - SyntheticModuleValue, - TypedDictEntry, - annotate_value, - extract_typevars, GenericValue, KnownValue, - make_coro_type, SubclassValue, + SyntheticModuleValue, + TypedDictEntry, TypedDictValue, TypedValue, TypeVarValue, - UNINITIALIZED_VALUE, Value, + annotate_value, + extract_typevars, + make_coro_type, ) PROPERTY_LIKE = {KnownValue(property), KnownValue(types.DynamicClassAttribute)}