make sigmatch use prepareNode for tyFromExpr (#24095)

fixes regression remaining after #24092

In #24092 `prepareNode` was updated so it wouldn't try to instantiate
generic type symbols (like `Generic` when `type Generic[T] = object`,
and `prepareNode` is what `tyFromExpr` uses in most of the compiler. An
exception is in sigmatch, which is now changed to use `prepareNode` to
make generic type symbols work in the same way as usual. However this
requires another change to work:

Dot fields and matches to `typedesc` on generic types generate
`tyFromExpr` in generic contexts since #24005, including generic type
symbols. But this means when we try to instantiate the `tyFromExpr` in
sigmatch, which increases `c.inGenericContext` for potentially remaining
unresolved expressions, dotcalls stay as `tyFromExpr` and so never
match. To fix this, we change the "generic type" check in dot fields and
`typedesc` matching to an "unresolved type" check which excludes generic
body types; and for generic body types, we only generate `tyFromExpr` if
the dot field is a generic parameter of the generic type (so that it
gets resolved only at instantiation).

Notes for the future:

* Sigmatch shouldn't have to `inc c.inGenericContext`, if a `tyFromExpr`
can't instantiate it's fine if we just fail the match (i.e. redirect the
instantiation errors from `semtypinst` to a match failure). Then again
maybe this is the best way to check for inability to instantiate.
* The `elif c.inGenericContext > 0 and t.containsUnresolvedType` check
in dotfields could maybe be simplified to just checking for `tyFromExpr`
and `tyGenericParam`, but I don't know if this is an exhaustive list.

4 files changed, 41 insertions, 5 deletions

diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index a83da5849..c90aa991b 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -617,7 +617,7 @@ proc semIs(c: PContext, n: PNode, flags: TExprFlags): PNode =
       n[1] = makeTypeSymNode(c, lhsType, n[1].info)
       lhsType = n[1].typ
   else:
-    if c.inGenericContext > 0 and lhsType.base.containsGenericType:
+    if c.inGenericContext > 0 and lhsType.base.containsUnresolvedType:
       # BUGFIX: don't evaluate this too early: ``T is void``
       return
 
@@ -1504,7 +1504,16 @@ proc tryReadingGenericParam(c: PContext, n: PNode, i: PIdent, t: PType): PNode =
       result.typ = makeTypeFromExpr(c, copyTree(result))
     else:
       result = nil
-  elif c.inGenericContext > 0 and t.containsGenericType:
+  of tyGenericBody, tyCompositeTypeClass:
+    if c.inGenericContext > 0:
+      result = readTypeParameter(c, t, i, n.info)
+      if result != nil:
+        # generic parameter exists, stop here but delay until instantiation
+        result = semGenericStmt(c, n)
+        result.typ = makeTypeFromExpr(c, copyTree(result))
+    else:
+      result = nil
+  elif c.inGenericContext > 0 and t.containsUnresolvedType:
     result = semGenericStmt(c, n)
     result.typ = makeTypeFromExpr(c, copyTree(result))
   else:
diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim
index b4fc319ec..759e8e6ab 100644
--- a/compiler/semtypinst.nim
+++ b/compiler/semtypinst.nim
@@ -801,6 +801,14 @@ proc replaceTypesInBody*(p: PContext, pt: TypeMapping, n: PNode;
   result = replaceTypeVarsN(cl, n, expectedType = expectedType)
   popInfoContext(p.config)
 
+proc prepareTypesInBody*(p: PContext, pt: TypeMapping, n: PNode;
+                         owner: PSym = nil): PNode =
+  var typeMap = initLayeredTypeMap(pt)
+  var cl = initTypeVars(p, typeMap, n.info, owner)
+  pushInfoContext(p.config, n.info)
+  result = prepareNode(cl, n)
+  popInfoContext(p.config)
+
 when false:
   # deadcode
   proc replaceTypesForLambda*(p: PContext, pt: TIdTable, n: PNode;
diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim
index e1f197679..17d2e7a4d 100644
--- a/compiler/sigmatch.nim
+++ b/compiler/sigmatch.nim
@@ -139,7 +139,7 @@ proc matchGenericParam(m: var TCandidate, formal: PType, n: PNode) =
     # don't match yet-unresolved generic instantiations
     while arg != nil and arg.kind == tyGenericParam:
       arg = idTableGet(m.bindings, arg)
-    if arg == nil or arg.containsGenericType:
+    if arg == nil or arg.containsUnresolvedType:
       m.state = csNoMatch
       return
   # fix up the type to get ready to match formal:
@@ -2048,7 +2048,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
       # proc foo(T: typedesc, x: T)
       # when `f` is an unresolved typedesc, `a` could be any
       # type, so we should not perform this check earlier
-      if c.c.inGenericContext > 0 and a.containsGenericType:
+      if c.c.inGenericContext > 0 and a.containsUnresolvedType:
         # generic type bodies can sometimes compile call expressions
         # prevent unresolved generic parameters from being passed to procs as
         # typedesc parameters
@@ -2087,7 +2087,9 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
       # also prevent infinite recursion below
       return isNone
     inc c.c.inGenericContext # to generate tyFromExpr again if unresolved
-    let reevaluated = tryResolvingStaticExpr(c, f.n, allowCalls = true).typ
+    # use prepareNode for consistency with other tyFromExpr in semtypinst:
+    let instantiated = prepareTypesInBody(c.c, c.bindings, f.n)
+    let reevaluated = c.c.semExpr(c.c, instantiated).typ
     dec c.c.inGenericContext
     case reevaluated.kind
     of tyFromExpr:
diff --git a/compiler/types.nim b/compiler/types.nim
index 2333672d2..60d812068 100644
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -1494,6 +1494,23 @@ proc containsGenericTypeIter(t: PType, closure: RootRef): bool =
 proc containsGenericType*(t: PType): bool =
   result = iterOverType(t, containsGenericTypeIter, nil)
 
+proc containsUnresolvedTypeIter(t: PType, closure: RootRef): bool =
+  if tfUnresolved in t.flags: return true
+  case t.kind
+  of tyStatic:
+    return t.n == nil
+  of tyTypeDesc:
+    if t.base.kind == tyNone: return true
+    if containsUnresolvedTypeIter(t.base, closure): return true
+    return false
+  of tyGenericInvocation, tyGenericParam, tyFromExpr, tyAnything:
+    return true
+  else:
+    return false
+
+proc containsUnresolvedType*(t: PType): bool =
+  result = iterOverType(t, containsUnresolvedTypeIter, nil)
+
 proc baseOfDistinct*(t: PType; g: ModuleGraph; idgen: IdGenerator): PType =
   if t.kind == tyDistinct:
     result = t.elementType