aboutsummaryrefslogtreecommitdiffstats
path: root/src/utils
diff options
context:
space:
mode:
authorbptato <nincsnevem662@gmail.com>2024-11-03 17:21:52 +0100
committerbptato <nincsnevem662@gmail.com>2024-11-03 17:21:52 +0100
commitcfc6bb4c3cb773b035fa54de2aa90efbdd47216e (patch)
tree8695e2d7654a3a145929465d688a273481efb047 /src/utils
parent21186a3f41bb0328dd34007d2f6300101e75a7cb (diff)
downloadchawan-cfc6bb4c3cb773b035fa54de2aa90efbdd47216e.tar.gz
utils, types: merge some modules
* line, vector, matrix -> path * twtuni, charcategory -> twtstr
Diffstat (limited to 'src/utils')
-rw-r--r--src/utils/charcategory.nim12
-rw-r--r--src/utils/luwrap.nim3
-rw-r--r--src/utils/regexutils.nim2
-rw-r--r--src/utils/strwidth.nim2
-rw-r--r--src/utils/twtstr.nim111
-rw-r--r--src/utils/twtuni.nim95
-rw-r--r--src/utils/widthconv.nim2
-rw-r--r--src/utils/wordbreak.nim2
8 files changed, 113 insertions, 116 deletions
diff --git a/src/utils/charcategory.nim b/src/utils/charcategory.nim
deleted file mode 100644
index e2c2a045..00000000
--- a/src/utils/charcategory.nim
+++ /dev/null
@@ -1,12 +0,0 @@
-const C0Controls* = {chr(0x00)..chr(0x1F)}
-const Controls* = (C0Controls + {chr(0x7F)})
-const Ascii* = {chr(0x00)..chr(0x7F)}
-const AsciiUpperAlpha* = {'A'..'Z'}
-const AsciiLowerAlpha* = {'a'..'z'}
-const AsciiAlpha* = (AsciiUpperAlpha + AsciiLowerAlpha)
-const NonAscii* = {char(0x80)..char(0xFF)}
-const AsciiDigit* = {'0'..'9'}
-const AsciiAlphaNumeric* = AsciiAlpha + AsciiDigit
-const AsciiOctDigit* = {'0'..'7'}
-const AsciiHexDigit* = (AsciiDigit + {'a'..'f', 'A'..'F'})
-const AsciiWhitespace* = {' ', '\n', '\r', '\t', '\f'}
diff --git a/src/utils/luwrap.nim b/src/utils/luwrap.nim
index 08b16e59..abe7e792 100644
--- a/src/utils/luwrap.nim
+++ b/src/utils/luwrap.nim
@@ -3,8 +3,7 @@ import std/strutils
import monoucha/libunicode
import monoucha/libregexp
-import utils/charcategory
-import utils/twtuni
+import utils/twtstr
proc passRealloc(opaque, p: pointer; size: csize_t): pointer {.cdecl.} =
return realloc(p, size)
diff --git a/src/utils/regexutils.nim b/src/utils/regexutils.nim
index 7c09625f..437d7839 100644
--- a/src/utils/regexutils.nim
+++ b/src/utils/regexutils.nim
@@ -3,7 +3,7 @@ import std/options
import monoucha/jsregex
import monoucha/libregexp
import types/opt
-import utils/charcategory
+import utils/twtstr
func countBackslashes(buf: string; i: int): int =
var j = 0
diff --git a/src/utils/strwidth.nim b/src/utils/strwidth.nim
index a651162b..7276c587 100644
--- a/src/utils/strwidth.nim
+++ b/src/utils/strwidth.nim
@@ -1,6 +1,6 @@
import utils/map
import utils/proptable
-import utils/twtuni
+import utils/twtstr
include res/map/charwidth_gen
diff --git a/src/utils/twtstr.nim b/src/utils/twtstr.nim
index 99e5f6ee..c53247e1 100644
--- a/src/utils/twtstr.nim
+++ b/src/utils/twtstr.nim
@@ -6,11 +6,116 @@ import std/posix
import std/strutils
import types/opt
-import utils/charcategory
import utils/map
-import utils/twtuni
-export charcategory
+const C0Controls* = {chr(0x00)..chr(0x1F)}
+const Controls* = (C0Controls + {chr(0x7F)})
+const Ascii* = {chr(0x00)..chr(0x7F)}
+const AsciiUpperAlpha* = {'A'..'Z'}
+const AsciiLowerAlpha* = {'a'..'z'}
+const AsciiAlpha* = (AsciiUpperAlpha + AsciiLowerAlpha)
+const NonAscii* = {char(0x80)..char(0xFF)}
+const AsciiDigit* = {'0'..'9'}
+const AsciiAlphaNumeric* = AsciiAlpha + AsciiDigit
+const AsciiOctDigit* = {'0'..'7'}
+const AsciiHexDigit* = (AsciiDigit + {'a'..'f', 'A'..'F'})
+const AsciiWhitespace* = {' ', '\n', '\r', '\t', '\f'}
+
+func nextUTF8*(s: openArray[char]; i: var int): uint32 =
+ let j = i
+ let u = uint32(s[j])
+ if u <= 0x7F:
+ inc i
+ return u
+ elif u shr 5 == 0b110:
+ let e = j + 2
+ if likely(e <= s.len):
+ i = e
+ return (u and 0x1F) shl 6 or
+ (uint32(s[j + 1]) and 0x3F)
+ elif u shr 4 == 0b1110:
+ let e = j + 3
+ if likely(e <= s.len):
+ i = e
+ return (u and 0xF) shl 12 or
+ (uint32(s[j + 1]) and 0x3F) shl 6 or
+ (uint32(s[j + 2]) and 0x3F)
+ elif u shr 3 == 0b11110:
+ let e = j + 4
+ if likely(e <= s.len):
+ i = e
+ return (u and 7) shl 18 or
+ (uint32(s[j + 1]) and 0x3F) shl 12 or
+ (uint32(s[j + 2]) and 0x3F) shl 6 or
+ (uint32(s[j + 3]) and 0x3F)
+ inc i
+ return 0xFFFD
+
+func prevUTF8*(s: openArray[char]; i: var int): uint32 =
+ var j = i - 1
+ while uint32(s[j]) shr 6 == 2:
+ dec j
+ i = j
+ return s.nextUTF8(j)
+
+func pointLenAt*(s: openArray[char]; i: int): int =
+ let u = uint8(s[i])
+ if u <= 0x7F:
+ return 1
+ elif u shr 5 == 0b110:
+ return 2
+ elif u shr 4 == 0b1110:
+ return 3
+ elif u shr 3 == 0b11110:
+ return 4
+ return 1
+
+iterator points*(s: openArray[char]): uint32 {.inline.} =
+ var i = 0
+ while i < s.len:
+ let u = s.nextUTF8(i)
+ yield u
+
+func toPoints*(s: openArray[char]): seq[uint32] =
+ result = @[]
+ for u in s.points:
+ result.add(u)
+
+proc addUTF8*(res: var string; u: uint32) =
+ if u < 0x80:
+ res &= char(u)
+ elif u < 0x800:
+ res &= char(u shr 6 or 0xC0)
+ res &= char(u and 0x3F or 0x80)
+ elif u < 0x10000:
+ res &= char(u shr 12 or 0xE0)
+ res &= char(u shr 6 and 0x3F or 0x80)
+ res &= char(u and 0x3F or 0x80)
+ else:
+ res &= char(u shr 18 or 0xF0)
+ res &= char(u shr 12 and 0x3F or 0x80)
+ res &= char(u shr 6 and 0x3F or 0x80)
+ res &= char(u and 0x3F or 0x80)
+
+func addUTF8*(res: var string; us: openArray[uint32]) =
+ for u in us:
+ res.addUTF8(u)
+
+func toUTF8*(u: uint32): string =
+ var s = ""
+ s.addUTF8(u)
+ return s
+
+func toUTF8*(us: openArray[uint32]): string =
+ var s = newStringOfCap(us.len shr 2)
+ s.addUTF8(us)
+ return s
+
+func pointLen*(s: openArray[char]): int =
+ var n = 0
+ for u in s.points:
+ inc n
+ return n
func onlyWhitespace*(s: string): bool =
return AllChars - AsciiWhitespace notin s
diff --git a/src/utils/twtuni.nim b/src/utils/twtuni.nim
deleted file mode 100644
index 47d37df6..00000000
--- a/src/utils/twtuni.nim
+++ /dev/null
@@ -1,95 +0,0 @@
-func nextUTF8*(s: openArray[char]; i: var int): uint32 =
- let j = i
- let u = uint32(s[j])
- if u <= 0x7F:
- inc i
- return u
- elif u shr 5 == 0b110:
- let e = j + 2
- if likely(e <= s.len):
- i = e
- return (u and 0x1F) shl 6 or
- (uint32(s[j + 1]) and 0x3F)
- elif u shr 4 == 0b1110:
- let e = j + 3
- if likely(e <= s.len):
- i = e
- return (u and 0xF) shl 12 or
- (uint32(s[j + 1]) and 0x3F) shl 6 or
- (uint32(s[j + 2]) and 0x3F)
- elif u shr 3 == 0b11110:
- let e = j + 4
- if likely(e <= s.len):
- i = e
- return (u and 7) shl 18 or
- (uint32(s[j + 1]) and 0x3F) shl 12 or
- (uint32(s[j + 2]) and 0x3F) shl 6 or
- (uint32(s[j + 3]) and 0x3F)
- inc i
- return 0xFFFD
-
-func prevUTF8*(s: openArray[char]; i: var int): uint32 =
- var j = i - 1
- while uint32(s[j]) shr 6 == 2:
- dec j
- i = j
- return s.nextUTF8(j)
-
-func pointLenAt*(s: openArray[char]; i: int): int =
- let u = uint8(s[i])
- if u <= 0x7F:
- return 1
- elif u shr 5 == 0b110:
- return 2
- elif u shr 4 == 0b1110:
- return 3
- elif u shr 3 == 0b11110:
- return 4
- return 1
-
-iterator points*(s: openArray[char]): uint32 {.inline.} =
- var i = 0
- while i < s.len:
- let u = s.nextUTF8(i)
- yield u
-
-func toPoints*(s: openArray[char]): seq[uint32] =
- result = @[]
- for u in s.points:
- result.add(u)
-
-proc addUTF8*(res: var string; u: uint32) =
- if u < 0x80:
- res &= char(u)
- elif u < 0x800:
- res &= char(u shr 6 or 0xC0)
- res &= char(u and 0x3F or 0x80)
- elif u < 0x10000:
- res &= char(u shr 12 or 0xE0)
- res &= char(u shr 6 and 0x3F or 0x80)
- res &= char(u and 0x3F or 0x80)
- else:
- res &= char(u shr 18 or 0xF0)
- res &= char(u shr 12 and 0x3F or 0x80)
- res &= char(u shr 6 and 0x3F or 0x80)
- res &= char(u and 0x3F or 0x80)
-
-func addUTF8*(res: var string; us: openArray[uint32]) =
- for u in us:
- res.addUTF8(u)
-
-func toUTF8*(u: uint32): string =
- var s = ""
- s.addUTF8(u)
- return s
-
-func toUTF8*(us: openArray[uint32]): string =
- var s = newStringOfCap(us.len shr 2)
- s.addUTF8(us)
- return s
-
-func pointLen*(s: openArray[char]): int =
- var n = 0
- for u in s.points:
- inc n
- return n
diff --git a/src/utils/widthconv.nim b/src/utils/widthconv.nim
index 32a904f6..a42ebf5d 100644
--- a/src/utils/widthconv.nim
+++ b/src/utils/widthconv.nim
@@ -1,7 +1,7 @@
import std/strutils
import utils/map
-import utils/twtuni
+import utils/twtstr
const CanHaveDakuten = ("かきくけこさしすせそたちつてとはひふへほカキクケコ" &
"サシスセソタチツテトハヒフヘホ").toPoints()
diff --git a/src/utils/wordbreak.nim b/src/utils/wordbreak.nim
index ff607fbe..84a5b26d 100644
--- a/src/utils/wordbreak.nim
+++ b/src/utils/wordbreak.nim
@@ -1,6 +1,6 @@
-import utils/charcategory
import utils/luwrap
import utils/strwidth
+import utils/twtstr
type BreakCategory* = enum
bcAlpha, bcSpace, bcSymbol, bcHan, bcHiragana, bcKatakana, bcHangul