move widestrs out of system (#20462)

* move widestrs out of system

* fix osproc

6 files changed, 239 insertions, 0 deletions

diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim
index d7706c17d..9748cd2ad 100644
--- a/lib/std/envvars.nim
+++ b/lib/std/envvars.nim
@@ -61,6 +61,8 @@ when not defined(nimscript):
       proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "<stdlib.h>".}
       from std/private/win_setenv import setEnvImpl
       import winlean
+      when defined(nimPreviewSlimSystem):
+        import std/widestrs
       proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv",
           header: "<stdlib.h>".}
       proc getEnvImpl(env: cstring): WideCString = c_wgetenv(env.newWideCString)
diff --git a/lib/std/oserrors.nim b/lib/std/oserrors.nim
index 9c2649eab..8d06c41da 100644
--- a/lib/std/oserrors.nim
+++ b/lib/std/oserrors.nim
@@ -16,6 +16,8 @@ type
 when not defined(nimscript):
   when defined(windows):
     import winlean
+    when defined(nimPreviewSlimSystem):
+      import std/widestrs
   else:
     var errno {.importc, header: "<errno.h>".}: cint
 
diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim
index 89bb0421f..9315f3e7e 100644
--- a/lib/std/private/win_setenv.nim
+++ b/lib/std/private/win_setenv.nim
@@ -23,6 +23,9 @@ check errno_t vs cint
 
 when not defined(windows): discard
 else:
+  when defined(nimPreviewSlimSystem):
+    import std/widestrs
+
   type wchar_t  {.importc: "wchar_t".} = int16
 
   proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {.
diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim
index fc132bba7..dadad9791 100644
--- a/lib/std/syncio.nim
+++ b/lib/std/syncio.nim
@@ -12,6 +12,8 @@
 include system/inclrtl
 import std/private/since
 import std/formatfloat
+when defined(windows):
+  import std/widestrs
 
 # ----------------- IO Part ------------------------------------------------
 type
diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim
index ee42f8a5c..1160aaaad 100644
--- a/lib/std/tempfiles.nim
+++ b/lib/std/tempfiles.nim
@@ -30,6 +30,8 @@ const
 
 when defined(windows):
   import winlean
+  when defined(nimPreviewSlimSystem):
+    import std/widestrs
 
   var O_RDWR {.importc: "_O_RDWR", header: "<fcntl.h>".}: cint
 
diff --git a/lib/std/widestrs.nim b/lib/std/widestrs.nim
new file mode 100644
index 000000000..7df4004d1
--- /dev/null
+++ b/lib/std/widestrs.nim
@@ -0,0 +1,228 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2012 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# Nim support for C/C++'s `wide strings`:idx:.
+
+#when not declared(ThisIsSystem):
+#  {.error: "You must not import this module explicitly".}
+
+type
+  Utf16Char* = distinct int16
+
+when defined(nimv2):
+
+  type
+    WideCString* = ptr UncheckedArray[Utf16Char]
+
+    WideCStringObj* = object
+      bytes: int
+      data: WideCString
+
+  proc `=destroy`(a: var WideCStringObj) =
+    if a.data != nil:
+      when compileOption("threads"):
+        deallocShared(a.data)
+      else:
+        dealloc(a.data)
+
+  proc `=copy`(a: var WideCStringObj; b: WideCStringObj) {.error.}
+
+  proc `=sink`(a: var WideCStringObj; b: WideCStringObj) =
+    a.bytes = b.bytes
+    a.data = b.data
+
+  proc createWide(a: var WideCStringObj; bytes: int) =
+    a.bytes = bytes
+    when compileOption("threads"):
+      a.data = cast[typeof(a.data)](allocShared0(bytes))
+    else:
+      a.data = cast[typeof(a.data)](alloc0(bytes))
+
+  template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx]
+  template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val
+
+  template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil)
+
+  converter toWideCString*(x: WideCStringObj): WideCString {.inline.} =
+    result = x.data
+
+else:
+  template nullWide(): untyped = nil
+
+  type
+    WideCString* = ref UncheckedArray[Utf16Char]
+    WideCStringObj* = WideCString
+
+  template createWide(a; L) =
+    unsafeNew(a, L)
+
+proc ord(arg: Utf16Char): int = int(cast[uint16](arg))
+
+proc len*(w: WideCString): int =
+  ## returns the length of a widestring. This traverses the whole string to
+  ## find the binary zero end marker!
+  result = 0
+  while int16(w[result]) != 0'i16: inc result
+
+const
+  UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16)
+  UNI_MAX_BMP = 0x0000FFFF
+  UNI_MAX_UTF16 = 0x0010FFFF
+  # UNI_MAX_UTF32 = 0x7FFFFFFF
+  # UNI_MAX_LEGAL_UTF32 = 0x0010FFFF
+
+  halfShift = 10
+  halfBase = 0x0010000
+  halfMask = 0x3FF
+
+  UNI_SUR_HIGH_START = 0xD800
+  UNI_SUR_HIGH_END = 0xDBFF
+  UNI_SUR_LOW_START = 0xDC00
+  UNI_SUR_LOW_END = 0xDFFF
+  UNI_REPL = 0xFFFD
+
+template ones(n: untyped): untyped = ((1 shl n)-1)
+
+template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) =
+  ## Returns the unicode character `s[i]` in `result`. If `doInc == true`
+  ## `i` is incremented by the number of bytes that have been processed.
+  bind ones
+
+  if ord(s[i]) <= 127:
+    result = ord(s[i])
+    when doInc: inc(i)
+  elif ord(s[i]) shr 5 == 0b110:
+    #assert(ord(s[i+1]) shr 6 == 0b10)
+    if i <= L - 2:
+      result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6))
+      when doInc: inc(i, 2)
+    else:
+      result = UNI_REPL
+      when doInc: inc(i)
+  elif ord(s[i]) shr 4 == 0b1110:
+    if i <= L - 3:
+      #assert(ord(s[i+1]) shr 6 == 0b10)
+      #assert(ord(s[i+2]) shr 6 == 0b10)
+      result = (ord(s[i]) and ones(4)) shl 12 or
+               (ord(s[i+1]) and ones(6)) shl 6 or
+               (ord(s[i+2]) and ones(6))
+      when doInc: inc(i, 3)
+    else:
+      result = UNI_REPL
+      when doInc: inc(i)
+  elif ord(s[i]) shr 3 == 0b11110:
+    if i <= L - 4:
+      #assert(ord(s[i+1]) shr 6 == 0b10)
+      #assert(ord(s[i+2]) shr 6 == 0b10)
+      #assert(ord(s[i+3]) shr 6 == 0b10)
+      result = (ord(s[i]) and ones(3)) shl 18 or
+               (ord(s[i+1]) and ones(6)) shl 12 or
+               (ord(s[i+2]) and ones(6)) shl 6 or
+               (ord(s[i+3]) and ones(6))
+      when doInc: inc(i, 4)
+    else:
+      result = UNI_REPL
+      when doInc: inc(i)
+  else:
+    result = 0xFFFD
+    when doInc: inc(i)
+
+iterator runes(s: cstring, L: int): int =
+  var
+    i = 0
+    result: int
+  while i < L:
+    fastRuneAt(s, i, L, result, true)
+    yield result
+
+proc newWideCString*(size: int): WideCStringObj =
+  createWide(result, size * 2 + 2)
+
+proc newWideCString*(source: cstring, L: int): WideCStringObj =
+  createWide(result, L * 2 + 2)
+  var d = 0
+  for ch in runes(source, L):
+
+    if ch <= UNI_MAX_BMP:
+      if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END:
+        result[d] = UNI_REPLACEMENT_CHAR
+      else:
+        result[d] = cast[Utf16Char](uint16(ch))
+    elif ch > UNI_MAX_UTF16:
+      result[d] = UNI_REPLACEMENT_CHAR
+    else:
+      let ch = ch - halfBase
+      result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START))
+      inc d
+      result[d] = cast[Utf16Char](uint16((ch and halfMask) + UNI_SUR_LOW_START))
+    inc d
+  result[d] = Utf16Char(0)
+
+proc newWideCString*(s: cstring): WideCStringObj =
+  if s.isNil: return nullWide
+
+  result = newWideCString(s, s.len)
+
+proc newWideCString*(s: string): WideCStringObj =
+  result = newWideCString(cstring s, s.len)
+
+proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string =
+  result = newStringOfCap(estimate + estimate shr 2)
+
+  var i = 0
+  while w[i].int16 != 0'i16:
+    var ch = ord(w[i])
+    inc i
+    if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END:
+      # If the 16 bits following the high surrogate are in the source buffer...
+      let ch2 = ord(w[i])
+
+      # If it's a low surrogate, convert to UTF32:
+      if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
+        ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
+        inc i
+      else:
+        #invalid UTF-16
+        ch = replacement
+    elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END:
+      #invalid UTF-16
+      ch = replacement
+
+    if ch < 0x80:
+      result.add chr(ch)
+    elif ch < 0x800:
+      result.add chr((ch shr 6) or 0xc0)
+      result.add chr((ch and 0x3f) or 0x80)
+    elif ch < 0x10000:
+      result.add chr((ch shr 12) or 0xe0)
+      result.add chr(((ch shr 6) and 0x3f) or 0x80)
+      result.add chr((ch and 0x3f) or 0x80)
+    elif ch <= 0x10FFFF:
+      result.add chr((ch shr 18) or 0xf0)
+      result.add chr(((ch shr 12) and 0x3f) or 0x80)
+      result.add chr(((ch shr 6) and 0x3f) or 0x80)
+      result.add chr((ch and 0x3f) or 0x80)
+    else:
+      # replacement char(in case user give very large number):
+      result.add chr(0xFFFD shr 12 or 0b1110_0000)
+      result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00)
+      result.add chr(0xFFFD and ones(6) or 0b10_0000_00)
+
+proc `$`*(s: WideCString): string =
+  result = s $ 80
+
+when defined(nimv2):
+  proc `$`*(s: WideCStringObj, estimate: int, replacement: int = 0xFFFD): string =
+    `$`(s.data, estimate, replacement)
+
+  proc `$`*(s: WideCStringObj): string =
+    $(s.data)
+
+  proc len*(w: WideCStringObj): int {.inline.} =
+    len(w.data)