generate symchoice for ambiguous types in templates & generics + handle types in symchoices (#23997)

fixes #23898, supersedes #23966 and #23990

Since #20631 ambiguous type symbols in templates are rejected outright,
now we generate a symchoice for type nodes if they're ambiguous, a
generalization of what was done in #22375. This is done for generics as
well. Symchoices also handle type symbols better now, ensuring their
type is a `typedesc` type; this probably isn't necessary for everything
to work but it makes the logic more robust.

Similar to #23989, we have to prepare for the fact that ambiguous type
symbols behave differently than normal type symbols and either error
normally or relegate to other routine symbols if the symbol is being
called. Generating a symchoice emulates this behavior, `semExpr` will
find the type symbol first, but since the symchoice has other symbols,
it will count as an ambiguous type symbol.

I know it seems spammy to carry around an ambiguity flag everywhere, but
in the future when we have something like #23104 we could just always
generate a symchoice, and the symchoice itself would carry the info of
whether the first symbol was ambiguous. But this could harm compiler
performance/memory use, it might be better to generate it only when we
have to, which in the case of type symbols is only when they're
ambiguous.

4 files changed, 53 insertions, 26 deletions

diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index 9293f8497..8332d404d 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -3131,12 +3131,16 @@ proc resolveIdentToSym(c: PContext, n: PNode, resultNode: var PNode,
     # unambiguous, or we don't care about ambiguity
     result = candidates[0]
   else:
-    # ambiguous symbols have 1 last chance as a symchoice,
-    # but type symbols cannot participate in symchoices
+    # ambiguous symbols have 1 last chance as a symchoice
     var choice = newNodeIT(nkClosedSymChoice, n.info, newTypeS(tyNone, c))
-    for c in candidates:
-      if c.kind notin {skType, skModule, skPackage}:
-        choice.add newSymNode(c, n.info)
+    for cand in candidates:
+      case cand.kind
+      of skModule, skPackage:
+        discard
+      of skType:
+        choice.add newSymNodeTypeDesc(cand, c.idgen, n.info)
+      else:
+        choice.add newSymNode(cand, n.info)
     if choice.len == 0:
       # we know candidates.len > 1, we just couldn't put any in a symchoice
       errorUseQualifier(c, n.info, candidates)
diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim
index e7f36879f..8efc8a94e 100644
--- a/compiler/semgnrc.nim
+++ b/compiler/semgnrc.nim
@@ -64,6 +64,7 @@ proc newOpenSym*(n: PNode): PNode {.inline.} =
 
 proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
                           ctx: var GenericCtx; flags: TSemGenericFlags,
+                          isAmbiguous: bool,
                           fromDotExpr=false): PNode =
   result = nil
   semIdeForTemplateOrGenericCheck(c.config, n, ctx.cursorInBody)
@@ -133,6 +134,11 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
   of skType:
     if (s.typ != nil) and
        (s.typ.flags * {tfGenericTypeParam, tfImplicitTypeParam} == {}):
+      if isAmbiguous:
+        # ambiguous types should be symchoices since lookup behaves
+        # differently for them in regular expressions
+        maybeDotChoice(c, n, s, fromDotExpr)
+        return
       result = newSymNodeTypeDesc(s, c.idgen, n.info)
       if canOpenSym(result.sym):
         if genericsOpenSym in c.features:
@@ -184,7 +190,7 @@ proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags,
     elif s.isMixedIn:
       result = symChoice(c, n, s, scForceOpen)
     else:
-      result = semGenericStmtSymbol(c, n, s, ctx, flags)
+      result = semGenericStmtSymbol(c, n, s, ctx, flags, amb)
   # else: leave as nkIdent
 
 proc newDot(n, b: PNode): PNode =
@@ -200,10 +206,11 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
 
   let luf = if withinMixin notin flags: {checkUndeclared, checkModule} else: {checkModule}
 
+  c.isAmbiguous = false
   var s = qualifiedLookUp(c, n, luf)
   if s != nil:
     isMacro = s.kind in {skTemplate, skMacro}
-    result = semGenericStmtSymbol(c, n, s, ctx, flags)
+    result = semGenericStmtSymbol(c, n, s, ctx, flags, c.isAmbiguous)
   else:
     n[0] = semGenericStmt(c, n[0], flags, ctx)
     result = n
@@ -217,24 +224,21 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
       isMacro = s.kind in {skTemplate, skMacro}
       if withinBind in flags or s.id in ctx.toBind:
         if s.kind == skType: # don't put types in sym choice
-          result = newDot(result, semGenericStmtSymbol(c, n, s, ctx, flags, fromDotExpr=true))
+          var ambig = false
+          if candidates.len > 1:
+            let s2 = searchInScopes(c, ident, ambig)
+          result = newDot(result, semGenericStmtSymbol(c, n, s, ctx, flags,
+            isAmbiguous = ambig, fromDotExpr = true))
         else:
           result = newDot(result, symChoice(c, n, s, scClosed))
       elif s.isMixedIn:
         result = newDot(result, symChoice(c, n, s, scForceOpen))
       else:
+        var ambig = false
         if s.kind == skType and candidates.len > 1:
-          var ambig = false
-          let s2 = searchInScopes(c, ident, ambig) 
-          if ambig:
-            # this is a type conversion like a.T where T is ambiguous with
-            # other types or routines
-            # in regular code, this never considers a type conversion and
-            # skips to routine overloading
-            # so symchoices are used which behave similarly with type symbols
-            result = newDot(result, symChoice(c, n, s, scForceOpen))
-            return
-        let syms = semGenericStmtSymbol(c, n, s, ctx, flags, fromDotExpr=true)
+          discard searchInScopes(c, ident, ambig)
+        let syms = semGenericStmtSymbol(c, n, s, ctx, flags,
+          isAmbiguous = ambig, fromDotExpr = true)
         result = newDot(result, syms)
 
 proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) =
@@ -301,7 +305,9 @@ proc semGenericStmt(c: PContext, n: PNode,
     # check if it is an expression macro:
     checkMinSonsLen(n, 1, c.config)
     let fn = n[0]
+    c.isAmbiguous = false
     var s = qualifiedLookUp(c, fn, {})
+    let ambig = c.isAmbiguous
     if s == nil and
         {withinMixin, withinConcept}*flags == {} and
         fn.kind in {nkIdent, nkAccQuoted} and
@@ -354,7 +360,12 @@ proc semGenericStmt(c: PContext, n: PNode,
       of skType:
         # bad hack for generics:
         if (s.typ != nil) and (s.typ.kind != tyGenericParam):
-          result[0] = newSymNodeTypeDesc(s, c.idgen, fn.info)
+          if ambig:
+            # ambiguous types should be symchoices since lookup behaves
+            # differently for them in regular expressions
+            result[0] = sc
+          else:
+            result[0] = newSymNodeTypeDesc(s, c.idgen, fn.info)
           onUse(fn.info, s)
           first = 1
       else:
diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim
index c6f2fb60c..f7653a890 100644
--- a/compiler/semtempl.nim
+++ b/compiler/semtempl.nim
@@ -218,7 +218,7 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
         if k == skParam and c.inTemplateHeader > 0:
           local.flags.incl sfTemplateParam
 
-proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode =
+proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField, isAmbiguous: bool): PNode =
   incl(s.flags, sfUsed)
   # bug #12885; ideally sem'checking is performed again afterwards marking
   # the symbol as used properly, but the nfSem mechanism currently prevents
@@ -245,6 +245,10 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode =
     result = n
   of skType:
     if isField and sfGenSym in s.flags: result = n
+    elif isAmbiguous:
+      # ambiguous types should be symchoices since lookup behaves
+      # differently for them in regular expressions
+      result = symChoice(c, n, s, scOpen, isField)
     else: result = newSymNodeTypeDesc(s, c.idgen, n.info)
   else:
     if isField and sfGenSym in s.flags: result = n
@@ -345,6 +349,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   case n.kind
   of nkIdent:
     if n.ident.id in c.toInject: return n
+    c.c.isAmbiguous = false
     let s = qualifiedLookUp(c.c, n, {})
     if s != nil:
       if s.owner == c.owner and s.kind == skParam and sfTemplateParam in s.flags:
@@ -362,9 +367,9 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
         result = newSymNode(s, n.info)
         onUse(n.info, s)
       else:
-        if s.kind in {skType, skVar, skLet, skConst}:
+        if s.kind in {skVar, skLet, skConst}:
           discard qualifiedLookUp(c.c, n, {checkAmbiguity, checkModule})
-        result = semTemplSymbol(c.c, n, s, c.noGenSym > 0)
+        result = semTemplSymbol(c.c, n, s, c.noGenSym > 0, c.c.isAmbiguous)
   of nkBind:
     result = semTemplBody(c, n[0])
   of nkBindStmt:
@@ -558,6 +563,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   of nkDotExpr, nkAccQuoted:
     # dotExpr is ambiguous: note that we explicitly allow 'x.TemplateParam',
     # so we use the generic code for nkDotExpr too
+    c.c.isAmbiguous = false
     let s = qualifiedLookUp(c.c, n, {})
     if s != nil:
       # mirror the nkIdent case
@@ -572,9 +578,9 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
       elif contains(c.toMixin, s.name.id):
         return symChoice(c.c, n, s, scForceOpen, c.noGenSym > 0)
       else:
-        if s.kind in {skType, skVar, skLet, skConst}:
+        if s.kind in {skVar, skLet, skConst}:
           discard qualifiedLookUp(c.c, n, {checkAmbiguity, checkModule})
-        return semTemplSymbol(c.c, n, s, c.noGenSym > 0)
+        return semTemplSymbol(c.c, n, s, c.noGenSym > 0, c.c.isAmbiguous)
     if n.kind == nkDotExpr:
       result = n
       result[0] = semTemplBody(c, n[0])
diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim
index f2865347f..d0c9ce029 100644
--- a/compiler/sigmatch.nim
+++ b/compiler/sigmatch.nim
@@ -2381,7 +2381,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     result = paramTypesMatchAux(m, f, a, arg, argOrig)
   else:
     # symbol kinds that don't participate in symchoice type disambiguation:
-    let matchSet = {low(TSymKind)..high(TSymKind)} - {skModule, skPackage, skType}
+    let matchSet = {low(TSymKind)..high(TSymKind)} - {skModule, skPackage}
 
     var best = -1
     result = arg
@@ -2426,6 +2426,12 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
         if arg[i].sym.kind in matchSet:
           copyCandidate(z, m)
           z.callee = arg[i].typ
+          if arg[i].sym.kind == skType and z.callee.kind != tyTypeDesc:
+            # creating the symchoice with the type sym having typedesc type
+            # breaks a lot of stuff, so we make the typedesc type here
+            # mirrored from `newSymNodeTypeDesc`
+            z.callee = newType(tyTypeDesc, c.idgen, arg[i].sym.owner)
+            z.callee.addSonSkipIntLit(arg[i].sym.typ, c.idgen)
           if tfUnresolved in z.callee.flags: continue
           z.calleeSym = arg[i].sym
           z.calleeScope = cmpScopes(m.c, arg[i].sym)