Fixes #20348; only respect the recursion limit if the symbol's generic type has been generated by the compiler (#20377)

Fixes #20348

3 files changed, 22 insertions, 12 deletions

diff --git a/compiler/ast.nim b/compiler/ast.nim
index d6e812f39..9e8d54432 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -232,7 +232,7 @@ type
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # 48 flags!
+  TSymFlag* = enum    # 49 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -304,6 +304,12 @@ type
     sfSingleUsedTemp  # For temporaries that we know will only be used once
     sfNoalias         # 'noalias' annotation, means C's 'restrict'
     sfEffectsDelayed  # an 'effectsDelayed' parameter
+    sfGeneratedType   # A anonymous generic type that is generated by the compiler for
+                      # objects that do not have generic parameters in case one of the
+                      # object fields has one.
+                      #
+                      # This is disallowed but can cause the typechecking to go into
+                      # an infinite loop, this flag is used as a sentinel to stop it.
 
   TSymFlags* = set[TSymFlag]
 
diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index 1e8930fc9..fd8f2180b 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -1425,6 +1425,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
         excl(objTy.flags, tfFinal)
       let obj = newSym(skType, getIdent(c.cache, s.name.s & ":ObjectType"),
                        nextSymId c.idgen, getCurrOwner(c), s.info)
+      obj.flags.incl sfGeneratedType
       let symNode = newSymNode(obj)
       obj.ast = a.shallowCopy
       case a[0].kind
diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim
index 504b83b4c..945667a81 100644
--- a/compiler/semtypinst.nim
+++ b/compiler/semtypinst.nim
@@ -499,17 +499,20 @@ proc propagateFieldFlags(t: PType, n: PNode) =
 
 proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
   template bailout =
-    if cl.recursionLimit > 100:
-      # bail out, see bug #2509. But note this caching is in general wrong,
-      # look at this example where TwoVectors should not share the generic
-      # instantiations (bug #3112):
-
-      # type
-      #   Vector[N: static[int]] = array[N, float64]
-      #   TwoVectors[Na, Nb: static[int]] = (Vector[Na], Vector[Nb])
-      result = PType(idTableGet(cl.localCache, t))
-      if result != nil: return result
-    inc cl.recursionLimit
+    if t.sym != nil and sfGeneratedType in t.sym.flags:
+      # Only consider the recursion limit if the symbol is a type with generic
+      # parameters that have not been explicitly supplied, typechecking should
+      # terminate when generic parameters are explicitly supplied.
+      if cl.recursionLimit > 100:
+        # bail out, see bug #2509. But note this caching is in general wrong,
+        # look at this example where TwoVectors should not share the generic
+        # instantiations (bug #3112):
+        # type
+        #   Vector[N: static[int]] = array[N, float64]
+        #   TwoVectors[Na, Nb: static[int]] = (Vector[Na], Vector[Nb])
+        result = PType(idTableGet(cl.localCache, t))
+        if result != nil: return result
+      inc cl.recursionLimit
 
   result = t
   if t == nil: return