Merge branch 'master' into closed.keyword

Author: alicederyn

misc/dump-ast.py

@@ -19,7 +19,7 @@ def dump(fname: str, python_version: tuple[int, int], quiet: bool = False) -> No
     options.python_version = python_version
     with open(fname, "rb") as f:
         s = f.read()
-        tree = parse(s, fname, None, errors=Errors(options), options=options, file_exists=True)
+        tree = parse(s, fname, None, errors=Errors(options), options=options)
         if not quiet:
             print(tree)
 

mypy/applytype.py

@@ -37,10 +37,12 @@ def get_target_type(
     report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None],
     context: Context,
     skip_unsatisfied: bool,
+    id_to_type: dict[TypeVarId, Type],
 ) -> Type | None:
     p_type = get_proper_type(type)
     if isinstance(p_type, UninhabitedType) and p_type.ambiguous and tvar.has_default():
-        return tvar.default
+        # Gradually expand defaults, as they may depend on previous type variables.
+        return expand_type(tvar.default, id_to_type)
     if isinstance(tvar, ParamSpecType):
         return type
     if isinstance(tvar, TypeVarTupleType):
@@ -113,7 +115,13 @@ def apply_generic_arguments(
             continue
 
         target_type = get_target_type(
-            tvar, type, callable, report_incompatible_typevar_value, context, skip_unsatisfied
+            tvar,
+            type,
+            callable,
+            report_incompatible_typevar_value,
+            context,
+            skip_unsatisfied,
+            id_to_type,
         )
         if target_type is not None:
             id_to_type[tvar.id] = target_type

mypy/build.py

@@ -1024,85 +1024,76 @@ def parse_all(self, states: list[State], post_parse: bool = True) -> None:
                 self.post_parse_all(states)
             return
 
-        sequential_states = []
         parallel_states = []
         for state in states:
+            if not self.fscache.exists(state.xpath, real_only=True):
+                state.source = state.get_source()
             if state.tree is not None:
                 # The file was already parsed.
-                continue
-            if not self.fscache.exists(state.xpath, real_only=True):
-                # New parser only supports parsing on-disk files.
-                sequential_states.append(state)
+                state.needs_parse = False
                 continue
             parallel_states.append(state)
+
         if len(parallel_states) > 1:
-            self.parse_parallel(sequential_states, parallel_states)
-        else:
-            # Avoid using executor when there is no parallelism.
-            for state in states:
-                state.parse_file()
-        if post_parse:
-            self.post_parse_all(states)
+            # This duplicates a bit of logic from State.parse_file(). This is done as an
+            # optimization to parallelize only those parts of the code that can be
+            # parallelized efficiently.
 
-    def parse_parallel(self, sequential_states: list[State], parallel_states: list[State]) -> None:
-        """Perform parallel parsing of states.
+            parallel_parsed_states, parallel_parsed_states_set = self.parse_files_threaded_raw(
+                parallel_states
+            )
 
-        Note: this duplicates a bit of logic from State.parse_file(). This is done
-        as an optimization to parallelize only those parts of the code that can be
-        parallelized efficiently.
-        """
-        parallel_parsed_states, parallel_parsed_states_set = self.parse_files_threaded_raw(
-            sequential_states, parallel_states
-        )
+            for state in parallel_parsed_states:
+                # New parser only returns serialized ASTs
+                with state.wrap_context():
+                    assert state.tree is not None
+                    raw_data = state.tree.raw_data
+                    if raw_data is not None:
+                        # Apply inline mypy config before deserialization, since
+                        # some options (e.g. implicit_optional) affect how the
+                        # AST is built during deserialization.
+                        state.source_hash = raw_data.source_hash
+                        state.apply_inline_configuration(raw_data.mypy_comments)
+                        state.tree = load_from_raw(
+                            state.xpath,
+                            state.id,
+                            raw_data,
+                            self.errors,
+                            state.options,
+                            imports_only=bool(self.workers),
+                        )
+                    if self.errors.is_blockers():
+                        self.log("Bailing due to parse errors")
+                        self.errors.raise_error()
 
-        for state in parallel_parsed_states:
-            # New parser returns serialized ASTs. Deserialize full trees only if not using
-            # parallel workers.
-            with state.wrap_context():
+            for state in parallel_states:
                 assert state.tree is not None
-                raw_data = state.tree.raw_data
-                if raw_data is not None:
-                    # Apply inline mypy config before deserialization, since
-                    # some options (e.g. implicit_optional) affect deserialization
-                    state.source_hash = raw_data.source_hash
-                    state.apply_inline_configuration(raw_data.mypy_comments)
-                    state.tree = load_from_raw(
-                        state.xpath,
-                        state.id,
-                        raw_data,
-                        self.errors,
-                        state.options,
-                        imports_only=bool(self.workers),
-                    )
-                if self.errors.is_blockers():
-                    self.log("Bailing due to parse errors")
-                    self.errors.raise_error()
-
-        for state in parallel_states:
-            assert state.tree is not None
-            if state in parallel_parsed_states_set:
+                if state in parallel_parsed_states_set:
+                    if state.tree.raw_data is not None:
+                        # source_hash was already extracted above, but raw_data
+                        # may have been preserved for workers (imports_only=True).
+                        pass
+                    elif state.source_hash is None:
+                        # At least namespace packages may not have source.
+                        state.get_source()
+                    state.early_errors = list(self.errors.error_info_map.get(state.xpath, []))
+                    state.semantic_analysis_pass1()
+                    self.ast_cache[state.id] = (state.tree, state.early_errors, state.source_hash)
+                self.modules[state.id] = state.tree
                 if state.tree.raw_data is not None:
-                    # source_hash was already extracted above, but raw_data
-                    # may have been preserved for workers (imports_only=True).
-                    pass
-                elif state.source_hash is None:
-                    # At least namespace packages may not have source.
-                    state.get_source()
-                state.early_errors = list(self.errors.error_info_map.get(state.xpath, []))
-                state.semantic_analysis_pass1()
-                self.ast_cache[state.id] = (state.tree, state.early_errors, state.source_hash)
-            self.modules[state.id] = state.tree
-            if state.tree.raw_data is not None:
-                state.size_hint = len(state.tree.raw_data.defs) + MIN_SIZE_HINT
-            state.check_blockers()
-            state.setup_errors()
-
-    def parse_files_threaded_raw(
-        self, sequential_states: list[State], parallel_states: list[State]
-    ) -> tuple[list[State], set[State]]:
-        """Parse files using a thread pool.
-
-        Also parse sequential states while waiting for the parallel results.
+                    state.size_hint = len(state.tree.raw_data.defs) + MIN_SIZE_HINT
+                state.check_blockers()
+                state.setup_errors()
+        elif len(parallel_states) == 1:
+            # Avoid using executor when there is no parallelism.
+            parallel_states[0].parse_file()
+
+        if post_parse:
+            self.post_parse_all(states)
+
+    def parse_files_threaded_raw(self, states: list[State]) -> tuple[list[State], set[State]]:
+        """Parse files in parallel using a thread pool.
+
         Trees from the new parser are left in raw (serialized) form.
 
         Return (list, set) of states that were actually parsed (not cached).
@@ -1118,25 +1109,21 @@ def parse_files_threaded_raw(
         # parse_file_inner() results in no visible improvement with more than 8 threads.
         # TODO: reuse thread pool and/or batch small files in single submit() call.
         with ThreadPoolExecutor(max_workers=min(available_threads, 8)) as executor:
-            for state in parallel_states:
+            for state in states:
                 state.needs_parse = False
                 if state.id not in self.ast_cache:
                     self.log(f"Parsing {state.xpath} ({state.id})")
                     ignore_errors = state.ignore_all or state.options.ignore_errors
                     if ignore_errors:
                         self.errors.ignored_files.add(state.xpath)
-                    futures.append(executor.submit(state.parse_file_inner, ""))
+                    futures.append(executor.submit(state.parse_file_inner, state.source))
                     parallel_parsed_states.append(state)
                     parallel_parsed_states_set.add(state)
                 else:
                     self.log(f"Using cached AST for {state.xpath} ({state.id})")
                     state.tree, state.early_errors, source_hash = self.ast_cache[state.id]
                     state.source_hash = source_hash
 
-            # Parse sequential before waiting on parallel.
-            for state in sequential_states:
-                state.parse_file()
-
             for fut in wait(futures).done:
                 fut.result()
 
@@ -1279,21 +1266,20 @@ def parse_file(
         self,
         id: str,
         path: str,
-        source: str,
+        source: str | None,
         options: Options,
         raw_data: FileRawData | None = None,
     ) -> MypyFile:
         """Parse the source of a file with the given name.
 
         Raise CompileError if there is a parse error.
         """
-        file_exists = self.fscache.exists(path, real_only=True)
         t0 = time.time()
         if raw_data:
             # If possible, deserialize from known binary data instead of parsing from scratch.
             tree = load_from_raw(path, id, raw_data, self.errors, options)
         else:
-            tree = parse(source, path, id, self.errors, options=options, file_exists=file_exists)
+            tree = parse(source, path, id, self.errors, options=options)
         tree._fullname = id
         if self.stats_enabled:
             with self.stats_lock:
@@ -3179,7 +3165,7 @@ def get_source(self) -> str:
                     else:
                         err = f"{self.path}: error: Cannot decode file: {str(decodeerr)}"
                     raise CompileError([err], module_with_blocker=self.id) from decodeerr
-            elif self.path and self.manager.fscache.isdir(self.path):
+            elif self.path and manager.fscache.isdir(self.path):
                 source = ""
                 self.source_hash = ""
             else:
@@ -3192,7 +3178,7 @@ def get_source(self) -> str:
         self.time_spent_us += time_spent_us(t0)
         return source
 
-    def parse_file_inner(self, source: str, raw_data: FileRawData | None = None) -> None:
+    def parse_file_inner(self, source: str | None, raw_data: FileRawData | None = None) -> None:
         t0 = time_ref()
         self.tree = self.manager.parse_file(
             self.id, self.xpath, source, options=self.options, raw_data=raw_data
@@ -3319,9 +3305,7 @@ def semantic_analysis_pass1(self) -> None:
         #
         # TODO: This should not be considered as a semantic analysis
         #     pass -- it's an independent pass.
-        if not options.native_parser or not self.manager.fscache.exists(
-            self.xpath, real_only=True
-        ):
+        if not options.native_parser:
             analyzer = SemanticAnalyzerPreAnalysis()
             with self.wrap_context():
                 analyzer.visit_file(self.tree, self.xpath, self.id, options)

mypy/checker.py

@@ -8581,6 +8581,9 @@ def lookup_fully_qualified_or_none(self, fullname: str, /) -> SymbolTableNode |
         except KeyError:
             return None
 
+    def record_fixed_type(self, fixed: TypeInfo | TypeAlias) -> None:
+        pass
+
     def fail(
         self,
         msg: str,
@@ -8629,6 +8632,9 @@ def is_func_scope(self) -> bool:
         # message types are ignored.
         return False
 
+    def is_nested_within_func_scope(self) -> bool:
+        return self._chk.scope.top_level_function() is not None
+
     @property
     def type(self) -> TypeInfo | None:
         return self._chk.scope.current_class()

mypy/checkexpr.py

@@ -949,7 +949,8 @@ def match_typeddict_call_with_dict(
         kwargs: list[tuple[Expression | None, Expression]],
         context: Context,
     ) -> bool:
-        result = self.validate_typeddict_kwargs(kwargs=kwargs, callee=callee)
+        with self.msg.filter_errors():
+            result = self.validate_typeddict_kwargs(kwargs=kwargs, callee=callee)
         if result is not None:
             validated_kwargs, _ = result
             return callee.required_keys <= set(validated_kwargs.keys()) <= set(callee.items.keys())
@@ -1764,6 +1765,28 @@ def check_callable_call(
             lambda i: self.accept(args[i]),
         )
 
+        if callee.special_sig == "tuple" and len(args) == 1:
+            with self.msg.filter_errors():
+                arg_type = get_proper_type(self.accept(args[0]))
+            # Give precise constructor signature for situations like this:
+            #     class Shape[*Ts](tuple[*Ts]): ...
+            #     Shape((1, 2))
+            # The argument type is the same as return type, but with builtins.tuple fallback.
+            if isinstance(arg_type, TupleType):
+                assert isinstance(callee.ret_type, ProperType)
+                if isinstance(callee.ret_type, TupleType):
+                    # Actual type argument is ignored by tuple_fallback() in this case.
+                    any_type = AnyType(TypeOfAny.special_form)
+                    new_arg_type = callee.ret_type.copy_modified(
+                        fallback=self.chk.named_generic_type("builtins.tuple", [any_type])
+                    )
+                    callee = callee.copy_modified(arg_types=[new_arg_type])
+                elif isinstance(callee.ret_type, Instance):
+                    new_arg_type = map_instance_to_supertype(
+                        callee.ret_type, self.chk.lookup_typeinfo("builtins.tuple")
+                    )
+                    callee = callee.copy_modified(arg_types=[new_arg_type])
+
         if callee.is_generic():
             need_refresh = any(
                 isinstance(v, (ParamSpecType, TypeVarTupleType)) for v in callee.variables
@@ -4932,10 +4955,11 @@ def visit_type_application(self, tapp: TypeApplication) -> Type:
             if tapp.expr.node.python_3_12_type_alias:
                 return self.type_alias_type_type()
             # Subscription of a (generic) alias in runtime context, expand the alias.
-            item = instantiate_type_alias(
+            item, _ = instantiate_type_alias(
                 tapp.expr.node,
                 tapp.types,
                 self.chk.fail,
+                self.chk.note,
                 tapp.expr.node.no_args,
                 tapp,
                 self.chk.options,
@@ -5000,17 +5024,16 @@ class LongName(Generic[T]): ...
         #     A = List[Tuple[T, T]]
         #     x = A() <- same as List[Tuple[Any, Any]], see PEP 484.
         disallow_any = self.chk.options.disallow_any_generics and self.is_callee
-        item = get_proper_type(
-            set_any_tvars(
-                alias,
-                [],
-                ctx.line,
-                ctx.column,
-                self.chk.options,
-                disallow_any=disallow_any,
-                fail=self.msg.fail,
-            )
+        item, _ = set_any_tvars(
+            alias,
+            [],
+            ctx.line,
+            ctx.column,
+            self.chk.options,
+            disallow_any=disallow_any,
+            fail=self.msg.fail,
         )
+        item = get_proper_type(item)
         if isinstance(item, Instance):
             # Normally we get a callable type (or overloaded) with .is_type_obj() true
             # representing the class's constructor
@@ -5073,11 +5096,7 @@ class C(Generic[T, Unpack[Ts]]): ...
                     return [AnyType(TypeOfAny.from_error)] * len(vars)
 
         # TODO: in future we may want to support type application to variadic functions.
-        if (
-            not vars
-            or not any(isinstance(v, TypeVarTupleType) for v in vars)
-            or not t.is_type_obj()
-        ):
+        if not vars or not t.is_type_obj() or t.type_object().fullname == "builtins.tuple":
             return list(args)
         info = t.type_object()
         # We reuse the logic from semanal phase to reduce code duplication.
@@ -5091,6 +5110,9 @@ class C(Generic[T, Unpack[Ts]]): ...
             )
             args = list(fake.args)
 
+        if not any(isinstance(v, TypeVarTupleType) for v in vars):
+            return args
+
         prefix = next(i for (i, v) in enumerate(vars) if isinstance(v, TypeVarTupleType))
         suffix = len(vars) - prefix - 1
         tvt = vars[prefix]

mypy/checkstrformat.py

@@ -587,7 +587,6 @@ def apply_field_accessors(
             module=None,
             options=self.chk.options,
             errors=temp_errors,
-            file_exists=False,
             eager=True,
         )
         if temp_errors.is_errors():