charwidth: use pre-generated map file

Also for reducing compilation time.

2 files changed, 42 insertions, 136 deletions

diff --git a/src/utils/proptable.nim b/src/utils/proptable.nim
new file mode 100644
index 00000000..a34459a8
--- /dev/null
+++ b/src/utils/proptable.nim
@@ -0,0 +1,40 @@
+import std/unicode
+
+# Lookup tables for characters on the BMP. This "only" takes up 8k of space
+# per table, as opposed to the 135k that storing all characters would require.
+# The downside is obviously that we need a binary search fallback for non-bmp.
+# We do not store a lookup table of ambiguous ranges, either.
+
+type
+  ptint* = int32
+  PropertyTable* = array[0x10000 div (sizeof(ptint) * 8), ptint]
+  RangeMap* = openArray[(uint32, uint32)]
+
+func makePropertyTable*(ranges: RangeMap, skip: RangeMap = @[]): PropertyTable =
+  var ucs: uint32 = 0
+  var j = 0
+  var k = 0
+  while ucs <= 0xFFFF:
+    if k > ranges.len:
+      break
+    if ranges[k][0] > ucs:
+      ucs = ranges[k][0]
+      continue
+    if ranges[k][1] < ucs:
+      inc k
+      continue
+    if j != skip.len and ucs == skip[j][0]:
+      ucs = skip[j][1] + 1
+      continue
+    let i = ucs div (sizeof(ptint) * 8)
+    let m = ucs mod (sizeof(ptint) * 8)
+    result[i] = result[i] or ptint(1 shl m)
+    inc ucs
+
+{.push boundChecks:off.}
+func contains*(props: PropertyTable, r: Rune): bool =
+  let u = ptint(r)
+  let i = u div (sizeof(ptint) * 8)
+  let m = u mod (sizeof(ptint) * 8)
+  return (props[i] and (1 shl m)) != 0
+{.pop.}
diff --git a/src/utils/strwidth.nim b/src/utils/strwidth.nim
index a27cb8b1..32e8ca7f 100644
--- a/src/utils/strwidth.nim
+++ b/src/utils/strwidth.nim
@@ -1,92 +1,13 @@
 import std/strutils
 import std/unicode
 
-import data/charwidth
+import utils/proptable
 import js/error
 import types/opt
 import utils/charcategory
 import utils/map
 
-# Combining chars from https://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
-#
-# The following two functions define the column width of an ISO 10646
-# character as follows:
-#
-#   - The null character (U+0000) has a column width of 0.
-#
-#   - Other C0/C1 control characters and DEL will lead to a return value of 0
-#
-#   - Non-spacing and enclosing combining characters (general category code Mn
-#     or Me in the Unicode database) have a column width of 0.
-#
-#   - SOFT HYPHEN (U+00AD) has a column width of 1.
-#
-#   - Other format characters (general category code Cf in the Unicode
-#     database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
-#
-#   - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF) have a
-#     column width of 0.
-#
-#   - Spacing characters in the East Asian Wide (W) or East Asian Full-width
-#     (F) category as defined in Unicode Technical Report #11 have a column
-#     width of 2.
-#
-#   - All remaining characters (including all printable ISO 8859-1 and WGL4
-#     characters, Unicode control characters, etc.) have a column width of 1.
-#
-
-# sorted list of non-overlapping intervals of non-spacing characters generated
-# by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c"
-const Combining = [
-  (0x0300u32, 0x036Fu32), (0x0483u32, 0x0486u32), (0x0488u32, 0x0489u32),
-  (0x0591u32, 0x05BDu32), (0x05BFu32, 0x05BFu32), (0x05C1u32, 0x05C2u32),
-  (0x05C4u32, 0x05C5u32), (0x05C7u32, 0x05C7u32), (0x0600u32, 0x0603u32),
-  (0x0610u32, 0x0615u32), (0x064Bu32, 0x065Eu32), (0x0670u32, 0x0670u32),
-  (0x06D6u32, 0x06E4u32), (0x06E7u32, 0x06E8u32), (0x06EAu32, 0x06EDu32),
-  (0x070Fu32, 0x070Fu32), (0x0711u32, 0x0711u32), (0x0730u32, 0x074Au32),
-  (0x07A6u32, 0x07B0u32), (0x07EBu32, 0x07F3u32), (0x0901u32, 0x0902u32),
-  (0x093Cu32, 0x093Cu32), (0x0941u32, 0x0948u32), (0x094Du32, 0x094Du32),
-  (0x0951u32, 0x0954u32), (0x0962u32, 0x0963u32), (0x0981u32, 0x0981u32),
-  (0x09BCu32, 0x09BCu32), (0x09C1u32, 0x09C4u32), (0x09CDu32, 0x09CDu32),
-  (0x09E2u32, 0x09E3u32), (0x0A01u32, 0x0A02u32), (0x0A3Cu32, 0x0A3Cu32),
-  (0x0A41u32, 0x0A42u32), (0x0A47u32, 0x0A48u32), (0x0A4Bu32, 0x0A4Du32),
-  (0x0A70u32, 0x0A71u32), (0x0A81u32, 0x0A82u32), (0x0ABCu32, 0x0ABCu32),
-  (0x0AC1u32, 0x0AC5u32), (0x0AC7u32, 0x0AC8u32), (0x0ACDu32, 0x0ACDu32),
-  (0x0AE2u32, 0x0AE3u32), (0x0B01u32, 0x0B01u32), (0x0B3Cu32, 0x0B3Cu32),
-  (0x0B3Fu32, 0x0B3Fu32), (0x0B41u32, 0x0B43u32), (0x0B4Du32, 0x0B4Du32),
-  (0x0B56u32, 0x0B56u32), (0x0B82u32, 0x0B82u32), (0x0BC0u32, 0x0BC0u32),
-  (0x0BCDu32, 0x0BCDu32), (0x0C3Eu32, 0x0C40u32), (0x0C46u32, 0x0C48u32),
-  (0x0C4Au32, 0x0C4Du32), (0x0C55u32, 0x0C56u32), (0x0CBCu32, 0x0CBCu32),
-  (0x0CBFu32, 0x0CBFu32), (0x0CC6u32, 0x0CC6u32), (0x0CCCu32, 0x0CCDu32),
-  (0x0CE2u32, 0x0CE3u32), (0x0D41u32, 0x0D43u32), (0x0D4Du32, 0x0D4Du32),
-  (0x0DCAu32, 0x0DCAu32), (0x0DD2u32, 0x0DD4u32), (0x0DD6u32, 0x0DD6u32),
-  (0x0E31u32, 0x0E31u32), (0x0E34u32, 0x0E3Au32), (0x0E47u32, 0x0E4Eu32),
-  (0x0EB1u32, 0x0EB1u32), (0x0EB4u32, 0x0EB9u32), (0x0EBBu32, 0x0EBCu32),
-  (0x0EC8u32, 0x0ECDu32), (0x0F18u32, 0x0F19u32), (0x0F35u32, 0x0F35u32),
-  (0x0F37u32, 0x0F37u32), (0x0F39u32, 0x0F39u32), (0x0F71u32, 0x0F7Eu32),
-  (0x0F80u32, 0x0F84u32), (0x0F86u32, 0x0F87u32), (0x0F90u32, 0x0F97u32),
-  (0x0F99u32, 0x0FBCu32), (0x0FC6u32, 0x0FC6u32), (0x102Du32, 0x1030u32),
-  (0x1032u32, 0x1032u32), (0x1036u32, 0x1037u32), (0x1039u32, 0x1039u32),
-  (0x1058u32, 0x1059u32), (0x1160u32, 0x11FFu32), (0x135Fu32, 0x135Fu32),
-  (0x1712u32, 0x1714u32), (0x1732u32, 0x1734u32), (0x1752u32, 0x1753u32),
-  (0x1772u32, 0x1773u32), (0x17B4u32, 0x17B5u32), (0x17B7u32, 0x17BDu32),
-  (0x17C6u32, 0x17C6u32), (0x17C9u32, 0x17D3u32), (0x17DDu32, 0x17DDu32),
-  (0x180Bu32, 0x180Du32), (0x18A9u32, 0x18A9u32), (0x1920u32, 0x1922u32),
-  (0x1927u32, 0x1928u32), (0x1932u32, 0x1932u32), (0x1939u32, 0x193Bu32),
-  (0x1A17u32, 0x1A18u32), (0x1B00u32, 0x1B03u32), (0x1B34u32, 0x1B34u32),
-  (0x1B36u32, 0x1B3Au32), (0x1B3Cu32, 0x1B3Cu32), (0x1B42u32, 0x1B42u32),
-  (0x1B6Bu32, 0x1B73u32), (0x1DC0u32, 0x1DCAu32), (0x1DFEu32, 0x1DFFu32),
-  (0x200Bu32, 0x200Fu32), (0x202Au32, 0x202Eu32), (0x2060u32, 0x2063u32),
-  (0x206Au32, 0x206Fu32), (0x20D0u32, 0x20EFu32), (0x302Au32, 0x302Fu32),
-  (0x3099u32, 0x309Au32), (0xA806u32, 0xA806u32), (0xA80Bu32, 0xA80Bu32),
-  (0xA825u32, 0xA826u32), (0xFB1Eu32, 0xFB1Eu32), (0xFE00u32, 0xFE0Fu32),
-  (0xFE20u32, 0xFE23u32), (0xFEFFu32, 0xFEFFu32), (0xFFF9u32, 0xFFFBu32),
-  (0x10A01u32, 0x10A03u32), (0x10A05u32, 0x10A06u32), (0x10A0Cu32, 0x10A0Fu32),
-  (0x10A38u32, 0x10A3Au32), (0x10A3Fu32, 0x10A3Fu32), (0x1D167u32, 0x1D169u32),
-  (0x1D173u32, 0x1D182u32), (0x1D185u32, 0x1D18Bu32), (0x1D1AAu32, 0x1D1ADu32),
-  (0x1D242u32, 0x1D244u32), (0xE0001u32, 0xE0001u32), (0xE0020u32, 0xE007Fu32),
-  (0xE0100u32, 0xE01EFu32)
-]
+include res/map/charwidth_gen
 
 func isDoubleWidthHigh(r: Rune): bool =
   return DoubleWidthRanges.isInRange(uint32(r))
@@ -100,66 +21,11 @@ func isDoubleWidthAmbiguousHigh(r: Rune): bool =
 func isCombining(r: Rune): bool =
   return Combining.isInRange(uint32(r))
 
-# Lookup tables for characters on the BMP. This "only" takes up 8k of space
-# per table, as opposed to the 135k that storing all characters would require.
-# The downside is obviously that we need a binary search fallback for non-bmp.
-# We do not store a lookup table of ambiguous ranges, either.
-
-static:
-  # following won't work on 16-bit
-  doAssert sizeof(int) >= sizeof(Rune)
-
-type PropertyTable = array[0x10000 div (sizeof(int) * 8), int]
-
-type RangeMap = openarray[(uint32, uint32)]
-
-func makePropertyTable(ranges: RangeMap, skip: RangeMap = @[]): PropertyTable =
-  var ucs: uint32 = 0
-  var j = 0
-  var k = 0
-  while ucs <= 0xFFFF:
-    if k > ranges.len:
-      break
-    if ranges[k][0] > ucs:
-      ucs = ranges[k][0]
-      continue
-    if ranges[k][1] < ucs:
-      inc k
-      continue
-    if j != skip.len and ucs == skip[j][0]:
-      ucs = skip[j][1] + 1
-      continue
-    let i = ucs div (sizeof(int) * 8)
-    let m = ucs mod (sizeof(int) * 8)
-    result[i] = result[i] or (1 shl m)
-    inc ucs
-
-const DoubleWidthTable = (func(): PropertyTable =
-  var ptab = makePropertyTable(DoubleWidthRanges, Combining)
-  # Control chars return a width of 2, and are displayed as ^{letter}.
-  for c in Controls:
-    let u = int(c)
-    let i = u div (sizeof(int) * 8)
-    let m = u mod (sizeof(int) * 8)
-    ptab[i] = ptab[i] or (1 shl m)
-  return ptab
-)()
-
-const CombiningTable = makePropertyTable(Combining)
-
 # One of the few global variables in the code. Honestly, it should not exist.
 var is_cjk_ambiguous = false
 proc set_cjk_ambiguous*(b: bool) =
   is_cjk_ambiguous = b
 
-{.push boundChecks:off.}
-func contains(props: PropertyTable, r: Rune): bool =
-  let u = int(r)
-  let i = u div (sizeof(int) * 8)
-  let m = u mod (sizeof(int) * 8)
-  return (props[i] and (1 shl m)) != 0
-{.pop.}
-
 # Warning: this shouldn't be called without normalization.
 # We could make this function more efficient in edge cases, but it's already
 # too complex for my taste.