right shift is now by default sign preserving (#11322)

* right shift is now by default sign preserving
* fix hashString and semfold
* enable arithmetic shift right globally for CI
* fix typo
* remove xxx
* use oldShiftRight as flag
* apply feedback
* add changelog entry

5 files changed, 49 insertions, 45 deletions

diff --git a/compiler/bitsets.nim b/compiler/bitsets.nim
index e0cf33b1d..d03a5915e 100644
--- a/compiler/bitsets.nim
+++ b/compiler/bitsets.nim
@@ -75,20 +75,20 @@ proc bitSetContains(x, y: TBitSet): bool =
 const populationCount: array[low(int8)..high(int8), int8] = block:
     var arr: array[low(int8)..high(int8), int8]
 
-    proc countSetBits(x: int8): int8 =
+    proc countSetBits(x: uint8): uint8 =
       return
-        ( x and 0b00000001'i8) +
-        ((x and 0b00000010'i8) shr 1) +
-        ((x and 0b00000100'i8) shr 2) +
-        ((x and 0b00001000'i8) shr 3) +
-        ((x and 0b00010000'i8) shr 4) +
-        ((x and 0b00100000'i8) shr 5) +
-        ((x and 0b01000000'i8) shr 6) +
-        ((x and 0b10000000'i8) shr 7)
+        ( x and 0b00000001'u8) +
+        ((x and 0b00000010'u8) shr 1) +
+        ((x and 0b00000100'u8) shr 2) +
+        ((x and 0b00001000'u8) shr 3) +
+        ((x and 0b00010000'u8) shr 4) +
+        ((x and 0b00100000'u8) shr 5) +
+        ((x and 0b01000000'u8) shr 6) +
+        ((x and 0b10000000'u8) shr 7)
 
 
     for it in low(int8)..high(int8):
-      arr[it] = countSetBits(it)
+      arr[it] = cast[int8](countSetBits(cast[uint8](it)))
 
     arr
 
diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim
index c608a8cb0..455012e60 100644
--- a/compiler/ccgutils.nim
+++ b/compiler/ccgutils.nim
@@ -28,29 +28,29 @@ proc stmtsContainPragma*(n: PNode, w: TSpecialWord): bool =
   result = getPragmaStmt(n, w) != nil
 
 proc hashString*(conf: ConfigRef; s: string): BiggestInt =
-  # has to be the same algorithm as system.hashString!
+  # has to be the same algorithm as strmantle.hashString!
   if CPU[conf.target.targetCPU].bit == 64:
     # we have to use the same bitwidth
     # as the target CPU
-    var b = 0'i64
+    var b = 0'u64
     for i in 0 ..< len(s):
-      b = b +% ord(s[i])
-      b = b +% `shl`(b, 10)
-      b = b xor `shr`(b, 6)
-    b = b +% `shl`(b, 3)
-    b = b xor `shr`(b, 11)
-    b = b +% `shl`(b, 15)
-    result = b
+      b = b + uint(s[i])
+      b = b + (b shl 10)
+      b = b xor (b shr 6)
+    b = b + (b shl 3)
+    b = b xor (b shr 11)
+    b = b + (b shl 15)
+    result = cast[Hash](b)
   else:
-    var a = 0'i32
+    var a = 0'u32
     for i in 0 ..< len(s):
-      a = a +% ord(s[i]).int32
-      a = a +% `shl`(a, 10'i32)
-      a = a xor `shr`(a, 6'i32)
-    a = a +% `shl`(a, 3'i32)
-    a = a xor `shr`(a, 11'i32)
-    a = a +% `shl`(a, 15'i32)
-    result = a
+      a = a + uint32(s[i])
+      a = a + (a shl 10)
+      a = a xor (a shr 6)
+    a = a + (a shl 3)
+    a = a xor (a shr 11)
+    a = a + (a shl 15)
+    result = cast[Hash](a)
 
 template getUniqueType*(key: PType): PType = key
 
diff --git a/compiler/lexer.nim b/compiler/lexer.nim
index 52559dad5..7533fb830 100644
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -647,34 +647,35 @@ proc handleDecChars(L: var TLexer, xi: var int) =
     inc(L.bufpos)
 
 proc addUnicodeCodePoint(s: var string, i: int) =
+  let i = cast[uint](i)
   # inlined toUTF-8 to avoid unicode and strutils dependencies.
   let pos = s.len
-  if i <=% 127:
+  if i <= 127:
     s.setLen(pos+1)
     s[pos+0] = chr(i)
-  elif i <=% 0x07FF:
+  elif i <= 0x07FF:
     s.setLen(pos+2)
     s[pos+0] = chr((i shr 6) or 0b110_00000)
     s[pos+1] = chr((i and ones(6)) or 0b10_0000_00)
-  elif i <=% 0xFFFF:
+  elif i <= 0xFFFF:
     s.setLen(pos+3)
     s[pos+0] = chr(i shr 12 or 0b1110_0000)
     s[pos+1] = chr(i shr 6 and ones(6) or 0b10_0000_00)
     s[pos+2] = chr(i and ones(6) or 0b10_0000_00)
-  elif i <=% 0x001FFFFF:
+  elif i <= 0x001FFFFF:
     s.setLen(pos+4)
     s[pos+0] = chr(i shr 18 or 0b1111_0000)
     s[pos+1] = chr(i shr 12 and ones(6) or 0b10_0000_00)
     s[pos+2] = chr(i shr 6 and ones(6) or 0b10_0000_00)
     s[pos+3] = chr(i and ones(6) or 0b10_0000_00)
-  elif i <=% 0x03FFFFFF:
+  elif i <= 0x03FFFFFF:
     s.setLen(pos+5)
     s[pos+0] = chr(i shr 24 or 0b111110_00)
     s[pos+1] = chr(i shr 18 and ones(6) or 0b10_0000_00)
     s[pos+2] = chr(i shr 12 and ones(6) or 0b10_0000_00)
     s[pos+3] = chr(i shr 6 and ones(6) or 0b10_0000_00)
     s[pos+4] = chr(i and ones(6) or 0b10_0000_00)
-  elif i <=% 0x7FFFFFFF:
+  elif i <= 0x7FFFFFFF:
     s.setLen(pos+6)
     s[pos+0] = chr(i shr 30 or 0b1111110_0)
     s[pos+1] = chr(i shr 24 and ones(6) or 0b10_0000_00)
diff --git a/compiler/semfold.nim b/compiler/semfold.nim
index e95184dfd..cb88ee6dd 100644
--- a/compiler/semfold.nim
+++ b/compiler/semfold.nim
@@ -248,13 +248,10 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode =
       result = doAndFit(newIntNodeT(`shl`(getInt(a), getInt(b)), n, g))
     else: internalError(g.config, n.info, "constant folding for shl")
   of mShrI:
-    case skipTypes(n.typ, abstractRange).kind
-    of tyInt8: result = newIntNodeT(int8(getInt(a)) shr int8(getInt(b)), n, g)
-    of tyInt16: result = newIntNodeT(int16(getInt(a)) shr int16(getInt(b)), n, g)
-    of tyInt32: result = newIntNodeT(int32(getInt(a)) shr int32(getInt(b)), n, g)
-    of tyInt64, tyInt, tyUInt..tyUInt64:
-      result = newIntNodeT(`shr`(getInt(a), getInt(b)), n, g)
-    else: internalError(g.config, n.info, "constant folding for shr")
+    let a = cast[uint64](getInt(a))
+    let b = cast[uint64](getInt(b))
+    let c = cast[BiggestInt](a shr b)
+    result = newIntNodeT(c, n, g)
   of mAshrI:
     case skipTypes(n.typ, abstractRange).kind
     of tyInt8: result = newIntNodeT(ashr(int8(getInt(a)), int8(getInt(b))), n, g)
diff --git a/compiler/vm.nim b/compiler/vm.nim
index aa8203a92..4f6208c59 100644
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -421,12 +421,15 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType):
       else:
         let srcDist = (sizeof(src.intVal) - srctyp.size) * 8
         let destDist = (sizeof(dest.intVal) - desttyp.size) * 8
+
+        var value = cast[BiggestUInt](src.intVal)
         when system.cpuEndian == bigEndian:
-          dest.intVal = (src.intVal shr srcDist) shl srcDist
-          dest.intVal = (dest.intVal shr destDist) shl destDist
+          value = (value shr srcDist) shl srcDist
+          value = (value shr destDist) shl destDist
         else:
-          dest.intVal = (src.intVal shl srcDist) shr srcDist
-          dest.intVal = (dest.intVal shl destDist) shr destDist
+          value = (value shl srcDist) shr srcDist
+          value = (value shl destDist) shr destDist
+        dest.intVal = cast[BiggestInt](value)
     of tyFloat..tyFloat64:
       if dest.kind != rkFloat:
         myreset(dest); dest.kind = rkFloat
@@ -818,7 +821,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       regs[ra].floatVal = regs[rb].floatVal / regs[rc].floatVal
     of opcShrInt:
       decodeBC(rkInt)
-      regs[ra].intVal = regs[rb].intVal shr regs[rc].intVal
+      let b = cast[uint64](regs[rb].intVal)
+      let c = cast[uint64](regs[rc].intVal)
+      let a = cast[int64](b shr c)
+      regs[ra].intVal = a
     of opcShlInt:
       decodeBC(rkInt)
       regs[ra].intVal = regs[rb].intVal shl regs[rc].intVal