fix #5643; fix #5644

4 files changed, 42 insertions, 7 deletions

diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim
index 9408a6d3d..d550a0c85 100644
--- a/compiler/semtypinst.nim
+++ b/compiler/semtypinst.nim
@@ -9,7 +9,7 @@
 
 # This module does the instantiation of generic types.
 
-import ast, astalgo, msgs, types, magicsys, semdata, renderer
+import ast, astalgo, msgs, types, magicsys, semdata, renderer, options
 
 const
   tfInstClearedFlags = {tfHasMeta, tfUnresolved}
@@ -50,6 +50,9 @@ proc searchInstTypes*(key: PType): PType =
       # types such as Channel[empty]. Why?
       # See the notes for PActor in handleGenericInvocation
       return
+    if not sameFlags(inst, key):
+      continue
+
     block matchType:
       for j in 1 .. high(key.sons):
         # XXX sameType is not really correct for nested generics?
@@ -247,10 +250,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
     result = PType(idTableGet(cl.localCache, t))
   else:
     result = searchInstTypes(t)
+
   if result != nil and eqTypeFlags*result.flags == eqTypeFlags*t.flags: return
   for i in countup(1, sonsLen(t) - 1):
     var x = t.sons[i]
-    if x.kind == tyGenericParam:
+    if x.kind in {tyGenericParam}:
       x = lookupTypeVar(cl, x)
       if x != nil:
         if header == t: header = instCopyType(cl, t)
diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim
index 56e12df03..2152e3652 100644
--- a/compiler/sigmatch.nim
+++ b/compiler/sigmatch.nim
@@ -1422,9 +1422,10 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
             internalAssert a.sons != nil and a.sons.len > 0
             c.typedescMatched = true
             var aa = a
-            while aa.kind in {tyTypeDesc, tyGenericParam} and
-                aa.len > 0:
+            while aa.kind in {tyTypeDesc, tyGenericParam} and aa.len > 0:
               aa = lastSon(aa)
+            if aa.kind == tyGenericParam:
+              return isGeneric
             result = typeRel(c, f.base, aa)
             if result > isGeneric: result = isGeneric
         else:
diff --git a/compiler/types.nim b/compiler/types.nim
index 3f84548a1..2886ac619 100644
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -886,6 +886,9 @@ proc isGenericAlias*(t: PType): bool =
 proc skipGenericAlias*(t: PType): PType =
   return if t.isGenericAlias: t.lastSon else: t
 
+proc sameFlags*(a, b: PType): bool {.inline.} =
+  result = eqTypeFlags*a.flags == eqTypeFlags*b.flags
+
 proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
   template cycleCheck() =
     # believe it or not, the direct check for ``containsOrIncl(c, a, b)``
@@ -898,9 +901,6 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     else:
       if containsOrIncl(c, a, b): return true
 
-  proc sameFlags(a, b: PType): bool {.inline.} =
-    result = eqTypeFlags*a.flags == eqTypeFlags*b.flags
-
   if x == y: return true
   var a = skipTypes(x, {tyGenericInst, tyAlias})
   var b = skipTypes(y, {tyGenericInst, tyAlias})
diff --git a/tests/generics/t5643.nim b/tests/generics/t5643.nim
new file mode 100644
index 000000000..962d5cef5
--- /dev/null
+++ b/tests/generics/t5643.nim
@@ -0,0 +1,30 @@
+type
+  Matrix*[M, N: static[int], T: SomeReal] = object
+    data: ref array[N * M, T]
+
+  Matrix64*[M, N: static[int]] = Matrix[M, N, float64]
+
+proc zeros64(M,N: static[int]): Matrix64[M,N] =
+  new result.data
+  for i in 0 .. < (M * N):
+    result.data[i] = 0'f64
+
+proc bar*[M,N: static[int], T](a: Matrix[M,N,T], b: Matrix[M,N,T]) =
+  discard
+
+let a = zeros64(2,2)
+bar(a,a)
+  # https://github.com/nim-lang/Nim/issues/5643
+  #
+  # The test case was failing here, because the compiler failed to
+  # detect the two matrix instantiations as the same type.
+  #
+  # The root cause was that the `T` type variable is a different
+  # type after the first Matrix type has been matched.
+  #
+  # Sigmatch was failing to match the second version of `T`, but
+  # due to some complex interplay between tyOr, tyTypeDesc and
+  # tyGenericParam this was allowed to went through. The generic
+  # instantiation of the second matrix was incomplete and the
+  # generic cache lookup failed, producing two separate types.
+