right shift is now by default sign preserving (#11322)

* right shift is now by default sign preserving
* fix hashString and semfold
* enable arithmetic shift right globally for CI
* fix typo
* remove xxx
* use oldShiftRight as flag
* apply feedback
* add changelog entry

3 files changed, 89 insertions, 84 deletions

diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim
index 226401b92..fb8b885dc 100644
--- a/lib/pure/collections/intsets.nim
+++ b/lib/pure/collections/intsets.nim
@@ -23,7 +23,7 @@ import
   hashes, math
 
 type
-  BitScalar = int
+  BitScalar = uint
 
 const
   InitIntSetSize = 8         # must be a power of two!
@@ -102,8 +102,8 @@ proc intSetPut(t: var IntSet, key: int): PTrunk =
 proc bitincl(s: var IntSet, key: int) {.inline.} =
   var t = intSetPut(s, `shr`(key, TrunkShift))
   var u = key and TrunkMask
-  t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or
-      `shl`(1, u and IntMask)
+  t.bits[u shr IntShift] = t.bits[u shr IntShift] or
+      (BitScalar(1) shl (u and IntMask))
 
 proc exclImpl(s: var IntSet, key: int) =
   if s.elems <= s.a.len:
@@ -113,11 +113,11 @@ proc exclImpl(s: var IntSet, key: int) =
         dec s.elems
         return
   else:
-    var t = intSetGet(s, `shr`(key, TrunkShift))
+    var t = intSetGet(s, key shr TrunkShift)
     if t != nil:
       var u = key and TrunkMask
-      t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] and
-          not `shl`(1, u and IntMask)
+      t.bits[u shr IntShift] = t.bits[u shr IntShift] and
+          not(BitScalar(1) shl (u and IntMask))
 
 template dollarImpl(): untyped =
   result = "{"
@@ -137,7 +137,7 @@ iterator items*(s: IntSet): int {.inline.} =
     while r != nil:
       var i = 0
       while i <= high(r.bits):
-        var w = r.bits[i]
+        var w: uint = r.bits[i]
         # taking a copy of r.bits[i] here is correct, because
         # modifying operations are not allowed during traversation
         var j = 0
@@ -186,7 +186,7 @@ proc contains*(s: IntSet, key: int): bool =
     var t = intSetGet(s, `shr`(key, TrunkShift))
     if t != nil:
       var u = key and TrunkMask
-      result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0
+      result = (t.bits[u shr IntShift] and (BitScalar(1) shl (u and IntMask))) != 0
     else:
       result = false
 
@@ -268,10 +268,10 @@ proc containsOrIncl*(s: var IntSet, key: int): bool =
     var t = intSetGet(s, `shr`(key, TrunkShift))
     if t != nil:
       var u = key and TrunkMask
-      result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0
+      result = (t.bits[u shr IntShift] and BitScalar(1) shl (u and IntMask)) != 0
       if not result:
-        t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or
-            `shl`(1, u and IntMask)
+        t.bits[u shr IntShift] = t.bits[u shr IntShift] or
+            (BitScalar(1) shl (u and IntMask))
     else:
       incl(s, key)
       result = false
diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim
index 763da523f..df29dc080 100644
--- a/lib/pure/hashes.nim
+++ b/lib/pure/hashes.nim
@@ -60,17 +60,22 @@ proc `!&`*(h: Hash, val: int): Hash {.inline.} =
   ## Mixes a hash value `h` with `val` to produce a new hash value.
   ##
   ## This is only needed if you need to implement a hash proc for a new datatype.
-  result = h +% val
-  result = result +% result shl 10
-  result = result xor (result shr 6)
+  let h = cast[uint](h)
+  let val = cast[uint](val)
+  var res = h + val
+  res = res + res shl 10
+  res = res xor (res shr 6)
+  result = cast[Hash](res)
 
 proc `!$`*(h: Hash): Hash {.inline.} =
   ## Finishes the computation of the hash value.
   ##
   ## This is only needed if you need to implement a hash proc for a new datatype.
-  result = h +% h shl 3
-  result = result xor (result shr 11)
-  result = result +% result shl 15
+  let h = cast[uint](h) # Hash is practically unsigned.
+  var res = h + h shl 3
+  res = res xor (res shr 11)
+  res = res + res shl 15
+  result = cast[Hash](res)
 
 proc hashData*(data: pointer, size: int): Hash =
   ## Hashes an array of bytes of size `size`.
@@ -105,7 +110,7 @@ proc hash*(x: pointer): Hash {.inline.} =
       }
     """
   else:
-    result = (cast[Hash](x)) shr 3 # skip the alignment
+    result = cast[Hash](cast[uint](x) shr 3) # skip the alignment
 
 when not defined(booting):
   proc hash*[T: proc](x: T): Hash {.inline.} =
diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim
index f38d89b95..83ec2783a 100644
--- a/lib/pure/unicode.nim
+++ b/lib/pure/unicode.nim
@@ -48,12 +48,12 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} =
 
   var i = 0
   while i < len(s):
-    if ord(s[i]) <=% 127: inc(i)
-    elif ord(s[i]) shr 5 == 0b110: inc(i, 2)
-    elif ord(s[i]) shr 4 == 0b1110: inc(i, 3)
-    elif ord(s[i]) shr 3 == 0b11110: inc(i, 4)
-    elif ord(s[i]) shr 2 == 0b111110: inc(i, 5)
-    elif ord(s[i]) shr 1 == 0b1111110: inc(i, 6)
+    if uint(s[i]) <= 127: inc(i)
+    elif uint(s[i]) shr 5 == 0b110: inc(i, 2)
+    elif uint(s[i]) shr 4 == 0b1110: inc(i, 3)
+    elif uint(s[i]) shr 3 == 0b11110: inc(i, 4)
+    elif uint(s[i]) shr 2 == 0b111110: inc(i, 5)
+    elif uint(s[i]) shr 1 == 0b1111110: inc(i, 6)
     else: inc i
     inc(result)
 
@@ -67,12 +67,12 @@ proc runeLenAt*(s: string, i: Natural): int =
     doAssert a.runeLenAt(0) == 1
     doAssert a.runeLenAt(1) == 2
 
-  if ord(s[i]) <=% 127: result = 1
-  elif ord(s[i]) shr 5 == 0b110: result = 2
-  elif ord(s[i]) shr 4 == 0b1110: result = 3
-  elif ord(s[i]) shr 3 == 0b11110: result = 4
-  elif ord(s[i]) shr 2 == 0b111110: result = 5
-  elif ord(s[i]) shr 1 == 0b1111110: result = 6
+  if uint(s[i]) <= 127: result = 1
+  elif uint(s[i]) shr 5 == 0b110: result = 2
+  elif uint(s[i]) shr 4 == 0b1110: result = 3
+  elif uint(s[i]) shr 3 == 0b11110: result = 4
+  elif uint(s[i]) shr 2 == 0b111110: result = 5
+  elif uint(s[i]) shr 1 == 0b1111110: result = 6
   else: result = 1
 
 const replRune = Rune(0xFFFD)
@@ -83,76 +83,76 @@ template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) =
   ## If ``doInc == true`` (default), ``i`` is incremented by the number
   ## of bytes that have been processed.
   bind ones
-  if ord(s[i]) <=% 127:
-    result = Rune(ord(s[i]))
+  if uint(s[i]) <= 127:
+    result = Rune(uint(s[i]))
     when doInc: inc(i)
-  elif ord(s[i]) shr 5 == 0b110:
-    # assert(ord(s[i+1]) shr 6 == 0b10)
+  elif uint(s[i]) shr 5 == 0b110:
+    # assert(uint(s[i+1]) shr 6 == 0b10)
     if i <= s.len - 2:
-      result = Rune((ord(s[i]) and (ones(5))) shl 6 or
-                    (ord(s[i+1]) and ones(6)))
+      result = Rune((uint(s[i]) and (ones(5))) shl 6 or
+                    (uint(s[i+1]) and ones(6)))
       when doInc: inc(i, 2)
     else:
       result = replRune
       when doInc: inc(i)
-  elif ord(s[i]) shr 4 == 0b1110:
-    # assert(ord(s[i+1]) shr 6 == 0b10)
-    # assert(ord(s[i+2]) shr 6 == 0b10)
+  elif uint(s[i]) shr 4 == 0b1110:
+    # assert(uint(s[i+1]) shr 6 == 0b10)
+    # assert(uint(s[i+2]) shr 6 == 0b10)
     if i <= s.len - 3:
-      result = Rune((ord(s[i]) and ones(4)) shl 12 or
-               (ord(s[i+1]) and ones(6)) shl 6 or
-               (ord(s[i+2]) and ones(6)))
+      result = Rune((uint(s[i]) and ones(4)) shl 12 or
+               (uint(s[i+1]) and ones(6)) shl 6 or
+               (uint(s[i+2]) and ones(6)))
       when doInc: inc(i, 3)
     else:
       result = replRune
       when doInc: inc(i)
-  elif ord(s[i]) shr 3 == 0b11110:
-    # assert(ord(s[i+1]) shr 6 == 0b10)
-    # assert(ord(s[i+2]) shr 6 == 0b10)
-    # assert(ord(s[i+3]) shr 6 == 0b10)
+  elif uint(s[i]) shr 3 == 0b11110:
+    # assert(uint(s[i+1]) shr 6 == 0b10)
+    # assert(uint(s[i+2]) shr 6 == 0b10)
+    # assert(uint(s[i+3]) shr 6 == 0b10)
     if i <= s.len - 4:
-      result = Rune((ord(s[i]) and ones(3)) shl 18 or
-               (ord(s[i+1]) and ones(6)) shl 12 or
-               (ord(s[i+2]) and ones(6)) shl 6 or
-               (ord(s[i+3]) and ones(6)))
+      result = Rune((uint(s[i]) and ones(3)) shl 18 or
+               (uint(s[i+1]) and ones(6)) shl 12 or
+               (uint(s[i+2]) and ones(6)) shl 6 or
+               (uint(s[i+3]) and ones(6)))
       when doInc: inc(i, 4)
     else:
       result = replRune
       when doInc: inc(i)
-  elif ord(s[i]) shr 2 == 0b111110:
-    # assert(ord(s[i+1]) shr 6 == 0b10)
-    # assert(ord(s[i+2]) shr 6 == 0b10)
-    # assert(ord(s[i+3]) shr 6 == 0b10)
-    # assert(ord(s[i+4]) shr 6 == 0b10)
+  elif uint(s[i]) shr 2 == 0b111110:
+    # assert(uint(s[i+1]) shr 6 == 0b10)
+    # assert(uint(s[i+2]) shr 6 == 0b10)
+    # assert(uint(s[i+3]) shr 6 == 0b10)
+    # assert(uint(s[i+4]) shr 6 == 0b10)
     if i <= s.len - 5:
-      result = Rune((ord(s[i]) and ones(2)) shl 24 or
-               (ord(s[i+1]) and ones(6)) shl 18 or
-               (ord(s[i+2]) and ones(6)) shl 12 or
-               (ord(s[i+3]) and ones(6)) shl 6 or
-               (ord(s[i+4]) and ones(6)))
+      result = Rune((uint(s[i]) and ones(2)) shl 24 or
+               (uint(s[i+1]) and ones(6)) shl 18 or
+               (uint(s[i+2]) and ones(6)) shl 12 or
+               (uint(s[i+3]) and ones(6)) shl 6 or
+               (uint(s[i+4]) and ones(6)))
       when doInc: inc(i, 5)
     else:
       result = replRune
       when doInc: inc(i)
-  elif ord(s[i]) shr 1 == 0b1111110:
-    # assert(ord(s[i+1]) shr 6 == 0b10)
-    # assert(ord(s[i+2]) shr 6 == 0b10)
-    # assert(ord(s[i+3]) shr 6 == 0b10)
-    # assert(ord(s[i+4]) shr 6 == 0b10)
-    # assert(ord(s[i+5]) shr 6 == 0b10)
+  elif uint(s[i]) shr 1 == 0b1111110:
+    # assert(uint(s[i+1]) shr 6 == 0b10)
+    # assert(uint(s[i+2]) shr 6 == 0b10)
+    # assert(uint(s[i+3]) shr 6 == 0b10)
+    # assert(uint(s[i+4]) shr 6 == 0b10)
+    # assert(uint(s[i+5]) shr 6 == 0b10)
     if i <= s.len - 6:
-      result = Rune((ord(s[i]) and ones(1)) shl 30 or
-               (ord(s[i+1]) and ones(6)) shl 24 or
-               (ord(s[i+2]) and ones(6)) shl 18 or
-               (ord(s[i+3]) and ones(6)) shl 12 or
-               (ord(s[i+4]) and ones(6)) shl 6 or
-               (ord(s[i+5]) and ones(6)))
+      result = Rune((uint(s[i]) and ones(1)) shl 30 or
+               (uint(s[i+1]) and ones(6)) shl 24 or
+               (uint(s[i+2]) and ones(6)) shl 18 or
+               (uint(s[i+3]) and ones(6)) shl 12 or
+               (uint(s[i+4]) and ones(6)) shl 6 or
+               (uint(s[i+5]) and ones(6)))
       when doInc: inc(i, 6)
     else:
       result = replRune
       when doInc: inc(i)
   else:
-    result = Rune(ord(s[i]))
+    result = Rune(uint(s[i]))
     when doInc: inc(i)
 
 proc runeAt*(s: string, i: Natural): Rune =
@@ -180,20 +180,20 @@ proc validateUTF8*(s: string): int =
   var i = 0
   let L = s.len
   while i < L:
-    if ord(s[i]) <=% 127:
+    if uint(s[i]) <= 127:
       inc(i)
-    elif ord(s[i]) shr 5 == 0b110:
-      if ord(s[i]) < 0xc2: return i # Catch overlong ascii representations.
-      if i+1 < L and ord(s[i+1]) shr 6 == 0b10: inc(i, 2)
+    elif uint(s[i]) shr 5 == 0b110:
+      if uint(s[i]) < 0xc2: return i # Catch overlong ascii representations.
+      if i+1 < L and uint(s[i+1]) shr 6 == 0b10: inc(i, 2)
       else: return i
-    elif ord(s[i]) shr 4 == 0b1110:
-      if i+2 < L and ord(s[i+1]) shr 6 == 0b10 and ord(s[i+2]) shr 6 == 0b10:
+    elif uint(s[i]) shr 4 == 0b1110:
+      if i+2 < L and uint(s[i+1]) shr 6 == 0b10 and uint(s[i+2]) shr 6 == 0b10:
         inc i, 3
       else: return i
-    elif ord(s[i]) shr 3 == 0b11110:
-      if i+3 < L and ord(s[i+1]) shr 6 == 0b10 and
-                     ord(s[i+2]) shr 6 == 0b10 and
-                     ord(s[i+3]) shr 6 == 0b10:
+    elif uint(s[i]) shr 3 == 0b11110:
+      if i+3 < L and uint(s[i+1]) shr 6 == 0b10 and
+                     uint(s[i+2]) shr 6 == 0b10 and
+                     uint(s[i+3]) shr 6 == 0b10:
         inc i, 4
       else: return i
     else:
@@ -906,7 +906,7 @@ proc lastRune*(s: string; last: int): (Rune, int) =
     result = (Rune(s[last]), 1)
   else:
     var L = 0
-    while last-L >= 0 and ord(s[last-L]) shr 6 == 0b10: inc(L)
+    while last-L >= 0 and uint(s[last-L]) shr 6 == 0b10: inc(L)
     var r: Rune
     fastRuneAt(s, last-L, r, false)
     result = (r, L+1)