fix: avoid recursive signature instantiation

crates/emmylua_code_analysis/src/compilation/test/for_range_var_infer_test.rs

@@ -211,6 +211,40 @@ mod test {
         assert_eq!(value_union.into_set(), expected_values);
     }
 
+    #[test]
+    fn test_pairs_self_iterator_method_does_not_recurse() {
+        let mut ws = VirtualWorkspace::new_with_init_std_lib();
+
+        ws.def(
+            r#"
+            ---@class Base
+            local base = {}
+            function base:iter()
+                return pairs(self)
+            end
+
+            ---@generic T
+            ---@param e T
+            ---@return T | Base
+            function Enum(e)
+            end
+
+            local Event = Enum {
+                A = 1,
+                B = 2,
+            }
+
+            for k, v in Event:iter() do
+                key_out = k
+                value_out = v
+            end
+        "#,
+        );
+
+        let _ = ws.expr_ty("key_out");
+        let _ = ws.expr_ty("value_out");
+    }
+
     #[test]
     fn test_issue_291() {
         let mut ws = VirtualWorkspace::new_with_init_std_lib();

crates/emmylua_code_analysis/src/semantic/generic/instantiate_type/instantiate_conditional_generic.rs

@@ -8,9 +8,7 @@ use crate::{
     semantic::{member::find_members_with_key, type_check::check_type_compact_with_level},
 };
 
-use super::{
-    get_default_constructor, instantiate_type_generic, instantiate_type_generic_with_context,
-};
+use super::{get_default_constructor, instantiate_type_generic_with_context};
 use crate::semantic::generic::type_substitutor::GenericInstantiateContext;
 
 #[derive(Debug, Clone, Copy)]
@@ -82,28 +80,19 @@ fn instantiate_conditional_once(
                 finalize_infer_assignments(infer_assignments),
             )
         } else {
-            instantiate_type_generic(
-                context.db,
-                conditional.get_false_type(),
-                context.substitutor,
-            )
+            instantiate_type_generic_with_context(context, conditional.get_false_type())
         };
     }
 
     match check_conditional_extends(context.db, &left_type, &right_type) {
         ConditionalCheck::True => instantiate_true_branch(context, conditional, HashMap::new()),
-        ConditionalCheck::False => instantiate_type_generic(
-            context.db,
-            conditional.get_false_type(),
-            context.substitutor,
-        ),
+        ConditionalCheck::False => {
+            instantiate_type_generic_with_context(context, conditional.get_false_type())
+        }
         ConditionalCheck::Both => {
             let true_type = instantiate_true_branch(context, conditional, HashMap::new());
-            let false_type = instantiate_type_generic(
-                context.db,
-                conditional.get_false_type(),
-                context.substitutor,
-            );
+            let false_type =
+                instantiate_type_generic_with_context(context, conditional.get_false_type());
             TypeOps::Union.apply(context.db, &true_type, &false_type)
         }
     }
@@ -163,18 +152,15 @@ fn instantiate_true_branch(
     infer_assignments: HashMap<GenericTplId, LuaType>,
 ) -> LuaType {
     if infer_assignments.is_empty() {
-        return instantiate_type_generic(
-            context.db,
-            conditional.get_true_type(),
-            context.substitutor,
-        );
+        return instantiate_type_generic_with_context(context, conditional.get_true_type());
     }
 
     let mut true_substitutor = context.substitutor.clone();
     for (tpl_id, ty) in infer_assignments {
         true_substitutor.insert_conditional_infer_type(tpl_id, ty);
     }
-    instantiate_type_generic(context.db, conditional.get_true_type(), &true_substitutor)
+    let true_context = context.with_substitutor(&true_substitutor);
+    instantiate_type_generic_with_context(&true_context, conditional.get_true_type())
 }
 
 fn contains_conditional_infer(ty: &LuaType) -> bool {

crates/emmylua_code_analysis/src/semantic/generic/instantiate_type/mod.rs

@@ -529,6 +529,13 @@ fn instantiate_signature(
     context: &GenericInstantiateContext,
     signature_id: &LuaSignatureId,
 ) -> LuaType {
+    // Substitution can make a signature mention itself again through its return
+    // type, e.g. `pairs(self)` on a table that also contains the method.
+    // Leave the nested occurrence opaque instead of expanding forever.
+    let Some(_signature_guard) = context.enter_signature(*signature_id) else {
+        return LuaType::Signature(*signature_id);
+    };
+
     if let Some(signature) = context.db.get_signature_index().get(signature_id) {
         let origin_type = {
             let fake_doc_function = signature.to_doc_func_type();

crates/emmylua_code_analysis/src/semantic/generic/type_substitutor.rs

@@ -1,7 +1,8 @@
 use hashbrown::{HashMap, HashSet};
+use std::{cell::RefCell, rc::Rc};
 
 use super::tpl_pattern::constant_decay;
-use crate::{DbIndex, GenericTplId, LuaType, LuaTypeDeclId};
+use crate::{DbIndex, GenericTplId, LuaSignatureId, LuaType, LuaTypeDeclId};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(super) enum UninferredTplPolicy {
@@ -16,6 +17,7 @@ pub struct GenericInstantiateContext<'a> {
     pub db: &'a DbIndex,
     pub substitutor: &'a TypeSubstitutor,
     policy: UninferredTplPolicy,
+    instantiating_signatures: Rc<RefCell<HashSet<LuaSignatureId>>>,
 }
 
 impl<'a> GenericInstantiateContext<'a> {
@@ -24,6 +26,7 @@ impl<'a> GenericInstantiateContext<'a> {
             db,
             substitutor,
             policy: UninferredTplPolicy::Fallback,
+            instantiating_signatures: Rc::new(RefCell::new(HashSet::new())),
         }
     }
 
@@ -32,6 +35,7 @@ impl<'a> GenericInstantiateContext<'a> {
             db: self.db,
             substitutor: self.substitutor,
             policy,
+            instantiating_signatures: self.instantiating_signatures.clone(),
         }
     }
 
@@ -43,12 +47,43 @@ impl<'a> GenericInstantiateContext<'a> {
             db: self.db,
             substitutor,
             policy: self.policy,
+            instantiating_signatures: self.instantiating_signatures.clone(),
         }
     }
 
     pub fn should_preserve_tpl_ref(&self) -> bool {
         self.policy == UninferredTplPolicy::PreserveTplRef
     }
+
+    pub(super) fn enter_signature(
+        &self,
+        signature_id: LuaSignatureId,
+    ) -> Option<InstantiatingSignatureGuard> {
+        if !self
+            .instantiating_signatures
+            .borrow_mut()
+            .insert(signature_id)
+        {
+            return None;
+        }
+
+        Some(InstantiatingSignatureGuard {
+            signatures: self.instantiating_signatures.clone(),
+            signature_id,
+        })
+    }
+}
+
+#[derive(Debug)]
+pub(super) struct InstantiatingSignatureGuard {
+    signatures: Rc<RefCell<HashSet<LuaSignatureId>>>,
+    signature_id: LuaSignatureId,
+}
+
+impl Drop for InstantiatingSignatureGuard {
+    fn drop(&mut self) {
+        self.signatures.borrow_mut().remove(&self.signature_id);
+    }
 }
 
 #[derive(Debug, Clone)]