Vm fix zero extend proc ze/ze64 && toU32/toU16/toU8 (#5988)

* fixes ze/ze64 procs in VM.
* fixes toU8/toU16/toU32.
* add tests for ze/ze64 toU32/toU16/toU8 procs

3 files changed, 29 insertions, 3 deletions

diff --git a/compiler/vm.nim b/compiler/vm.nim
index 908bbeb69..2dd6d6a9c 100644
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -1483,6 +1483,17 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       createStrKeepNode(regs[ra])
       if regs[ra].node.strVal.isNil: regs[ra].node.strVal = newStringOfCap(1000)
       storeAny(regs[ra].node.strVal, typ, regs[rb].regToNode)
+    of opcToNarrowInt:
+      decodeBC(rkInt)
+      let mask = (1'i64 shl rc) - 1 # 0xFF
+      let signbit = 1'i64 shl (rc - 1) # 0x80
+      let toggle = mask - signbit # 0x7F
+      # algorithm: -((i8 and 0xFF) xor 0x7F) + 0x7F
+      # mask off higher bits.
+      # uses two's complement to sign-extend integer.
+      # reajust integer into desired range.
+      regs[ra].intVal = -((regs[rb].intVal and mask) xor toggle) + toggle
+
     inc pc
 
 proc execute(c: PCtx, start: int): PNode =
diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim
index 5045f3f07..7e1309e0a 100644
--- a/compiler/vmdef.nim
+++ b/compiler/vmdef.nim
@@ -136,7 +136,8 @@ type
     opcNBindSym,
     opcSetType,   # dest.typ = types[Bx]
     opcTypeTrait,
-    opcMarshalLoad, opcMarshalStore
+    opcMarshalLoad, opcMarshalStore,
+    opcToNarrowInt
 
   TBlock* = object
     label*: PSym
diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim
index d65fe22e9..ab969a42f 100644
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -875,11 +875,25 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mBitnotI:
     genUnaryABC(c, n, dest, opcBitnotInt)
     genNarrowU(c, n, dest)
-  of mZe8ToI, mZe8ToI64, mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64,
-     mToU8, mToU16, mToU32, mToFloat, mToBiggestFloat, mToInt,
+  of mToFloat, mToBiggestFloat, mToInt,
      mToBiggestInt, mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr,
      mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr:
     genConv(c, n, n.sons[1], dest)
+  of mZe8ToI, mZe8ToI64, mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64:
+    #genNarrowU modified
+    let t = skipTypes(n.sons[1].typ, abstractVar-{tyTypeDesc})
+    let tmp = c.genx(n.sons[1])
+    c.gABC(n, opcNarrowU, tmp, TRegister(t.size*8))
+    # assign result to dest register
+    if dest < 0: dest = c.getTemp(n.typ)
+    c.gABC(n, opcAsgnInt, dest, tmp)
+    c.freeTemp(tmp)
+  of mToU8, mToU16, mToU32:
+    let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
+    var tmp = c.genx(n.sons[1])
+    if dest < 0: dest = c.getTemp(n.typ)
+    c.gABC(n, opcToNarrowInt, dest, tmp, TRegister(t.size*8))
+    c.freeTemp(tmp)
   of mEqStr, mEqCString: genBinaryABC(c, n, dest, opcEqStr)
   of mLeStr: genBinaryABC(c, n, dest, opcLeStr)
   of mLtStr: genBinaryABC(c, n, dest, opcLtStr)