Fix various issues exposed by typeshed-client update (#615)

Author: JelleZijlstra

docs/changelog.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+- Add support for `__new__` methods returning `typing.Self`, fixing
+  various failures with the latest release of `typeshed-client` (#615)
+- Add support for importing stub-only modules in other stubs (#615)
+- Fix signature compatibility bug involving `**kwargs` and positional-only
+  arguments (#615)
 - Fix type narrowing with `in` on enum types in the negative case (#606)
 - Fix crash when `getattr()` on a module object throws an error (#603)
 - Fix handling of positional-only arguments using `/` syntax in stubs (#601)

pyanalyze/analysis_lib.py

@@ -6,6 +6,7 @@
 import ast
 import linecache
 import os
+from pathlib import Path
 import secrets
 import sys
 import types
@@ -14,7 +15,7 @@
 
 
 def _all_files(
-    root: str, filter_function: Optional[Callable[[str], bool]] = None
+    root: Path, filter_function: Optional[Callable[[str], bool]] = None
 ) -> Set[str]:
     """Returns the set of all files at the given root.
 
@@ -30,7 +31,7 @@ def _all_files(
     return all_files
 
 
-def files_with_extension_from_directory(extension: str, dirname: str) -> Set[str]:
+def files_with_extension_from_directory(extension: str, dirname: Path) -> Set[str]:
     """Finds all files in a given directory with this extension."""
     return _all_files(dirname, filter_function=lambda fn: fn.endswith("." + extension))
 

pyanalyze/arg_spec.py

@@ -644,12 +644,13 @@ def _uncached_get_argspec(
             signature = self._cached_get_argspec(
                 _ENUM_CALL, impl, is_asynq, in_overload_resolution
             )
-            bound_sig = make_bound_method(
-                signature, Composite(SubclassValue(TypedValue(obj)))
-            )
+            self_value = SubclassValue(TypedValue(obj))
+            bound_sig = make_bound_method(signature, Composite(self_value))
             if bound_sig is None:
                 return None
-            sig = bound_sig.get_signature(preserve_impl=True, ctx=self.ctx)
+            sig = bound_sig.get_signature(
+                preserve_impl=True, ctx=self.ctx, self_annotation_value=self_value
+            )
             if sig is not None:
                 return sig
             return bound_sig
@@ -739,6 +740,7 @@ def _uncached_get_argspec(
         if inspect.isclass(obj):
             obj = UnwrapClass.unwrap(obj, self.options)
             override = ConstructorHooks.get_constructor(obj, self.options)
+            is_dunder_new = False
             if isinstance(override, Signature):
                 signature = override
             else:
@@ -760,6 +762,7 @@ def _uncached_get_argspec(
                     # doesn't have a useful signature.
                     # In practice, we saw this make a difference with NamedTuples.
                     elif isinstance(obj.__new__, FunctionType):
+                        is_dunder_new = True
                         constructor = obj.__new__
                     else:
                         constructor = obj.__init__
@@ -783,7 +786,15 @@ def _uncached_get_argspec(
             bound_sig = make_bound_method(signature, Composite(TypedValue(obj)))
             if bound_sig is None:
                 return None
-            sig = bound_sig.get_signature(preserve_impl=True, ctx=self.ctx)
+            if is_dunder_new:
+                self_annotation_value = KnownValue(obj)
+            else:
+                self_annotation_value = TypedValue(obj)
+            sig = bound_sig.get_signature(
+                preserve_impl=True,
+                ctx=self.ctx,
+                self_annotation_value=self_annotation_value,
+            )
             if sig is not None:
                 return sig
             return bound_sig

pyanalyze/attributes.py

@@ -30,6 +30,7 @@
     KnownValue,
     KnownValueWithTypeVars,
     MultiValuedValue,
+    SyntheticModuleValue,
     set_self,
     SubclassValue,
     TypedValue,
@@ -67,6 +68,9 @@ def record_attr_read(self, obj: Any) -> None:
     def get_property_type_from_argspec(self, obj: property) -> Value:
         return AnyValue(AnySource.inference)
 
+    def resolve_name_from_typeshed(self, module: str, name: str) -> Value:
+        return UNINITIALIZED_VALUE
+
     def get_attribute_from_typeshed(self, typ: type, *, on_class: bool) -> Value:
         return UNINITIALIZED_VALUE
 
@@ -130,6 +134,9 @@ def get_attribute(ctx: AttrContext) -> Value:
         attribute_value = AnyValue(AnySource.from_another)
     elif isinstance(root_value, MultiValuedValue):
         raise TypeError("caller should unwrap MultiValuedValue")
+    elif isinstance(root_value, SyntheticModuleValue):
+        module = ".".join(root_value.module_path)
+        attribute_value = ctx.resolve_name_from_typeshed(module, ctx.attr)
     else:
         attribute_value = UNINITIALIZED_VALUE
     if (

pyanalyze/checker.py

@@ -239,6 +239,8 @@ def display_value(self, value: Value) -> str:
             sig = self.arg_spec_cache.get_argspec(value.val)
         elif isinstance(value, UnboundMethodValue):
             sig = value.get_signature(self)
+        elif isinstance(value, SubclassValue) and value.exactly:
+            sig = self.signature_from_value(value)
         else:
             sig = None
         if sig is not None:
@@ -439,6 +441,9 @@ def _extract_protocol_members(typ: type) -> Set[str]:
 class CheckerAttrContext(AttrContext):
     checker: Checker
 
+    def resolve_name_from_typeshed(self, module: str, name: str) -> Value:
+        return self.checker.ts_finder.resolve_name(module, name)
+
     def get_attribute_from_typeshed(self, typ: type, *, on_class: bool) -> Value:
         return self.checker.ts_finder.get_attribute(typ, self.attr, on_class=on_class)
 

pyanalyze/node_visitor.py

@@ -14,6 +14,7 @@
 import logging
 import os
 import os.path
+from pathlib import Path
 import re
 import subprocess
 import sys
@@ -1116,7 +1117,7 @@ def _get_all_files(lst: Iterable[str]) -> Iterable[str]:
     for entry in lst:
         if os.path.isdir(entry):
             yield from sorted(
-                analysis_lib.files_with_extension_from_directory("py", entry)
+                analysis_lib.files_with_extension_from_directory("py", Path(entry))
             )
         else:
             yield entry

pyanalyze/options.py

@@ -362,7 +362,9 @@ def parse_config_file(
     if path in seen_paths:
         raise InvalidConfigOption("Recursive config inclusion detected")
     with path.open("rb") as f:
-        data = tomli.load(f)
+        # tomli annotates the arg as BinaryIO, and we don't treat BufferedReader
+        # as a BinaryIO
+        data = tomli.load(f)  # static analysis: ignore[incompatible_argument]
     data = data.get("tool", {}).get("pyanalyze", {})
     yield from _parse_config_section(
         data, path=path, priority=priority, seen_paths={path, *seen_paths}

pyanalyze/signature.py

@@ -61,6 +61,7 @@
 )
 from .typevar import resolve_bounds_map
 from .value import (
+    SelfT,
     annotate_value,
     AnnotatedValue,
     AnySource,
@@ -1453,6 +1454,7 @@ def can_assign(
         if their_ellipsis is not None:
             args_annotation = kwargs_annotation = AnyValue(AnySource.ellipsis_callable)
         consumed_positional = set()
+        consumed_required_pos_only = set()
         consumed_keyword = set()
         consumed_paramspec = False
         for i, my_param in enumerate(self.parameters.values()):
@@ -1479,6 +1481,8 @@ def can_assign(
                         )
                     tv_maps.append(tv_map)
                     consumed_positional.add(their_params[i].name)
+                    if their_params[i].default is None:
+                        consumed_required_pos_only.add(their_params[i].name)
                 elif args_annotation is not None:
                     new_tv_maps = can_assign_var_positional(
                         my_param, args_annotation, i - their_args_index, ctx
@@ -1608,6 +1612,7 @@ def can_assign(
                     if param.name not in consumed_keyword
                     and param.kind
                     in (ParameterKind.KEYWORD_ONLY, ParameterKind.POSITIONAL_OR_KEYWORD)
+                    and param.name not in consumed_required_pos_only
                 ]
                 for extra_param in extra_keyword:
                     tv_map = extra_param.get_annotation().can_assign(my_annotation, ctx)
@@ -1885,6 +1890,7 @@ def bind_self(
         self,
         *,
         preserve_impl: bool = False,
+        self_annotation_value: Optional[Value] = None,
         self_value: Optional[Value] = None,
         ctx: CanAssignContext,
     ) -> Optional["Signature"]:
@@ -1910,12 +1916,14 @@ def bind_self(
             self_annotation = params[0].annotation
         else:
             return None
-        if self_value is not None:
-            tv_map = get_tv_map(self_annotation, self_value, ctx)
+        if self_annotation_value is not None:
+            tv_map = get_tv_map(self_annotation, self_annotation_value, ctx)
             if isinstance(tv_map, CanAssignError):
                 return None
         else:
             tv_map = {}
+        if self_value is not None:
+            tv_map = {**tv_map, SelfT: self_value}
         if tv_map:
             new_params = {
                 param.name: param.substitute_typevars(tv_map) for param in new_params
@@ -2414,10 +2422,16 @@ def bind_self(
         *,
         preserve_impl: bool = False,
         self_value: Optional[Value] = None,
+        self_annotation_value: Optional[Value] = None,
         ctx: CanAssignContext,
     ) -> Optional["ConcreteSignature"]:
         bound_sigs = [
-            sig.bind_self(preserve_impl=preserve_impl, self_value=self_value, ctx=ctx)
+            sig.bind_self(
+                preserve_impl=preserve_impl,
+                self_value=self_value,
+                self_annotation_value=self_annotation_value,
+                ctx=ctx,
+            )
             for sig in self.signatures
         ]
         bound_sigs = [sig for sig in bound_sigs if isinstance(sig, Signature)]
@@ -2496,10 +2510,19 @@ def check_call(
         return ret
 
     def get_signature(
-        self, *, preserve_impl: bool = False, ctx: CanAssignContext
+        self,
+        *,
+        preserve_impl: bool = False,
+        ctx: CanAssignContext,
+        self_annotation_value: Optional[Value] = None,
     ) -> Optional[ConcreteSignature]:
+        if self_annotation_value is None:
+            self_annotation_value = self.self_composite.value
         return self.signature.bind_self(
-            preserve_impl=preserve_impl, self_value=self.self_composite.value, ctx=ctx
+            preserve_impl=preserve_impl,
+            self_value=self.self_composite.value,
+            ctx=ctx,
+            self_annotation_value=self_annotation_value,
         )
 
     def has_return_value(self) -> bool:

pyanalyze/stubs/_pyanalyze_tests-stubs/self.pyi

@@ -4,5 +4,6 @@ class X:
     def ret(self) -> Self: ...
     @classmethod
     def from_config(cls) -> Self: ...
+    def __new__(cls) -> Self: ...
 
 class Y(X): ...

pyanalyze/stubs/_pyanalyze_tests-stubs/tsself.pyi

@@ -0,0 +1,5 @@
+import _typeshed
+
+class X:
+    def __new__(cls: type[_typeshed.Self]) -> _typeshed.Self: ...
+    def method(self) -> int: ...