diff options
| author | bptato <nincsnevem662@gmail.com> | 2024-11-03 17:21:52 +0100 |
|---|---|---|
| committer | bptato <nincsnevem662@gmail.com> | 2024-11-03 17:21:52 +0100 |
| commit | cfc6bb4c3cb773b035fa54de2aa90efbdd47216e (patch) | |
| tree | 8695e2d7654a3a145929465d688a273481efb047 /src/utils | |
| parent | 21186a3f41bb0328dd34007d2f6300101e75a7cb (diff) | |
| download | chawan-cfc6bb4c3cb773b035fa54de2aa90efbdd47216e.tar.gz | |
utils, types: merge some modules
* line, vector, matrix -> path
* twtuni, charcategory -> twtstr
Diffstat (limited to 'src/utils')
| -rw-r--r-- | src/utils/charcategory.nim | 12 | ||||
| -rw-r--r-- | src/utils/luwrap.nim | 3 | ||||
| -rw-r--r-- | src/utils/regexutils.nim | 2 | ||||
| -rw-r--r-- | src/utils/strwidth.nim | 2 | ||||
| -rw-r--r-- | src/utils/twtstr.nim | 111 | ||||
| -rw-r--r-- | src/utils/twtuni.nim | 95 | ||||
| -rw-r--r-- | src/utils/widthconv.nim | 2 | ||||
| -rw-r--r-- | src/utils/wordbreak.nim | 2 |
8 files changed, 113 insertions, 116 deletions
diff --git a/src/utils/charcategory.nim b/src/utils/charcategory.nim deleted file mode 100644 index e2c2a045..00000000 --- a/src/utils/charcategory.nim +++ /dev/null @@ -1,12 +0,0 @@ -const C0Controls* = {chr(0x00)..chr(0x1F)} -const Controls* = (C0Controls + {chr(0x7F)}) -const Ascii* = {chr(0x00)..chr(0x7F)} -const AsciiUpperAlpha* = {'A'..'Z'} -const AsciiLowerAlpha* = {'a'..'z'} -const AsciiAlpha* = (AsciiUpperAlpha + AsciiLowerAlpha) -const NonAscii* = {char(0x80)..char(0xFF)} -const AsciiDigit* = {'0'..'9'} -const AsciiAlphaNumeric* = AsciiAlpha + AsciiDigit -const AsciiOctDigit* = {'0'..'7'} -const AsciiHexDigit* = (AsciiDigit + {'a'..'f', 'A'..'F'}) -const AsciiWhitespace* = {' ', '\n', '\r', '\t', '\f'} diff --git a/src/utils/luwrap.nim b/src/utils/luwrap.nim index 08b16e59..abe7e792 100644 --- a/src/utils/luwrap.nim +++ b/src/utils/luwrap.nim @@ -3,8 +3,7 @@ import std/strutils import monoucha/libunicode import monoucha/libregexp -import utils/charcategory -import utils/twtuni +import utils/twtstr proc passRealloc(opaque, p: pointer; size: csize_t): pointer {.cdecl.} = return realloc(p, size) diff --git a/src/utils/regexutils.nim b/src/utils/regexutils.nim index 7c09625f..437d7839 100644 --- a/src/utils/regexutils.nim +++ b/src/utils/regexutils.nim @@ -3,7 +3,7 @@ import std/options import monoucha/jsregex import monoucha/libregexp import types/opt -import utils/charcategory +import utils/twtstr func countBackslashes(buf: string; i: int): int = var j = 0 diff --git a/src/utils/strwidth.nim b/src/utils/strwidth.nim index a651162b..7276c587 100644 --- a/src/utils/strwidth.nim +++ b/src/utils/strwidth.nim @@ -1,6 +1,6 @@ import utils/map import utils/proptable -import utils/twtuni +import utils/twtstr include res/map/charwidth_gen diff --git a/src/utils/twtstr.nim b/src/utils/twtstr.nim index 99e5f6ee..c53247e1 100644 --- a/src/utils/twtstr.nim +++ b/src/utils/twtstr.nim @@ -6,11 +6,116 @@ import std/posix import std/strutils import types/opt -import utils/charcategory import utils/map -import utils/twtuni -export charcategory +const C0Controls* = {chr(0x00)..chr(0x1F)} +const Controls* = (C0Controls + {chr(0x7F)}) +const Ascii* = {chr(0x00)..chr(0x7F)} +const AsciiUpperAlpha* = {'A'..'Z'} +const AsciiLowerAlpha* = {'a'..'z'} +const AsciiAlpha* = (AsciiUpperAlpha + AsciiLowerAlpha) +const NonAscii* = {char(0x80)..char(0xFF)} +const AsciiDigit* = {'0'..'9'} +const AsciiAlphaNumeric* = AsciiAlpha + AsciiDigit +const AsciiOctDigit* = {'0'..'7'} +const AsciiHexDigit* = (AsciiDigit + {'a'..'f', 'A'..'F'}) +const AsciiWhitespace* = {' ', '\n', '\r', '\t', '\f'} + +func nextUTF8*(s: openArray[char]; i: var int): uint32 = + let j = i + let u = uint32(s[j]) + if u <= 0x7F: + inc i + return u + elif u shr 5 == 0b110: + let e = j + 2 + if likely(e <= s.len): + i = e + return (u and 0x1F) shl 6 or + (uint32(s[j + 1]) and 0x3F) + elif u shr 4 == 0b1110: + let e = j + 3 + if likely(e <= s.len): + i = e + return (u and 0xF) shl 12 or + (uint32(s[j + 1]) and 0x3F) shl 6 or + (uint32(s[j + 2]) and 0x3F) + elif u shr 3 == 0b11110: + let e = j + 4 + if likely(e <= s.len): + i = e + return (u and 7) shl 18 or + (uint32(s[j + 1]) and 0x3F) shl 12 or + (uint32(s[j + 2]) and 0x3F) shl 6 or + (uint32(s[j + 3]) and 0x3F) + inc i + return 0xFFFD + +func prevUTF8*(s: openArray[char]; i: var int): uint32 = + var j = i - 1 + while uint32(s[j]) shr 6 == 2: + dec j + i = j + return s.nextUTF8(j) + +func pointLenAt*(s: openArray[char]; i: int): int = + let u = uint8(s[i]) + if u <= 0x7F: + return 1 + elif u shr 5 == 0b110: + return 2 + elif u shr 4 == 0b1110: + return 3 + elif u shr 3 == 0b11110: + return 4 + return 1 + +iterator points*(s: openArray[char]): uint32 {.inline.} = + var i = 0 + while i < s.len: + let u = s.nextUTF8(i) + yield u + +func toPoints*(s: openArray[char]): seq[uint32] = + result = @[] + for u in s.points: + result.add(u) + +proc addUTF8*(res: var string; u: uint32) = + if u < 0x80: + res &= char(u) + elif u < 0x800: + res &= char(u shr 6 or 0xC0) + res &= char(u and 0x3F or 0x80) + elif u < 0x10000: + res &= char(u shr 12 or 0xE0) + res &= char(u shr 6 and 0x3F or 0x80) + res &= char(u and 0x3F or 0x80) + else: + res &= char(u shr 18 or 0xF0) + res &= char(u shr 12 and 0x3F or 0x80) + res &= char(u shr 6 and 0x3F or 0x80) + res &= char(u and 0x3F or 0x80) + +func addUTF8*(res: var string; us: openArray[uint32]) = + for u in us: + res.addUTF8(u) + +func toUTF8*(u: uint32): string = + var s = "" + s.addUTF8(u) + return s + +func toUTF8*(us: openArray[uint32]): string = + var s = newStringOfCap(us.len shr 2) + s.addUTF8(us) + return s + +func pointLen*(s: openArray[char]): int = + var n = 0 + for u in s.points: + inc n + return n func onlyWhitespace*(s: string): bool = return AllChars - AsciiWhitespace notin s diff --git a/src/utils/twtuni.nim b/src/utils/twtuni.nim deleted file mode 100644 index 47d37df6..00000000 --- a/src/utils/twtuni.nim +++ /dev/null @@ -1,95 +0,0 @@ -func nextUTF8*(s: openArray[char]; i: var int): uint32 = - let j = i - let u = uint32(s[j]) - if u <= 0x7F: - inc i - return u - elif u shr 5 == 0b110: - let e = j + 2 - if likely(e <= s.len): - i = e - return (u and 0x1F) shl 6 or - (uint32(s[j + 1]) and 0x3F) - elif u shr 4 == 0b1110: - let e = j + 3 - if likely(e <= s.len): - i = e - return (u and 0xF) shl 12 or - (uint32(s[j + 1]) and 0x3F) shl 6 or - (uint32(s[j + 2]) and 0x3F) - elif u shr 3 == 0b11110: - let e = j + 4 - if likely(e <= s.len): - i = e - return (u and 7) shl 18 or - (uint32(s[j + 1]) and 0x3F) shl 12 or - (uint32(s[j + 2]) and 0x3F) shl 6 or - (uint32(s[j + 3]) and 0x3F) - inc i - return 0xFFFD - -func prevUTF8*(s: openArray[char]; i: var int): uint32 = - var j = i - 1 - while uint32(s[j]) shr 6 == 2: - dec j - i = j - return s.nextUTF8(j) - -func pointLenAt*(s: openArray[char]; i: int): int = - let u = uint8(s[i]) - if u <= 0x7F: - return 1 - elif u shr 5 == 0b110: - return 2 - elif u shr 4 == 0b1110: - return 3 - elif u shr 3 == 0b11110: - return 4 - return 1 - -iterator points*(s: openArray[char]): uint32 {.inline.} = - var i = 0 - while i < s.len: - let u = s.nextUTF8(i) - yield u - -func toPoints*(s: openArray[char]): seq[uint32] = - result = @[] - for u in s.points: - result.add(u) - -proc addUTF8*(res: var string; u: uint32) = - if u < 0x80: - res &= char(u) - elif u < 0x800: - res &= char(u shr 6 or 0xC0) - res &= char(u and 0x3F or 0x80) - elif u < 0x10000: - res &= char(u shr 12 or 0xE0) - res &= char(u shr 6 and 0x3F or 0x80) - res &= char(u and 0x3F or 0x80) - else: - res &= char(u shr 18 or 0xF0) - res &= char(u shr 12 and 0x3F or 0x80) - res &= char(u shr 6 and 0x3F or 0x80) - res &= char(u and 0x3F or 0x80) - -func addUTF8*(res: var string; us: openArray[uint32]) = - for u in us: - res.addUTF8(u) - -func toUTF8*(u: uint32): string = - var s = "" - s.addUTF8(u) - return s - -func toUTF8*(us: openArray[uint32]): string = - var s = newStringOfCap(us.len shr 2) - s.addUTF8(us) - return s - -func pointLen*(s: openArray[char]): int = - var n = 0 - for u in s.points: - inc n - return n diff --git a/src/utils/widthconv.nim b/src/utils/widthconv.nim index 32a904f6..a42ebf5d 100644 --- a/src/utils/widthconv.nim +++ b/src/utils/widthconv.nim @@ -1,7 +1,7 @@ import std/strutils import utils/map -import utils/twtuni +import utils/twtstr const CanHaveDakuten = ("かきくけこさしすせそたちつてとはひふへほカキクケコ" & "サシスセソタチツテトハヒフヘホ").toPoints() diff --git a/src/utils/wordbreak.nim b/src/utils/wordbreak.nim index ff607fbe..84a5b26d 100644 --- a/src/utils/wordbreak.nim +++ b/src/utils/wordbreak.nim @@ -1,6 +1,6 @@ -import utils/charcategory import utils/luwrap import utils/strwidth +import utils/twtstr type BreakCategory* = enum bcAlpha, bcSpace, bcSymbol, bcHan, bcHiragana, bcKatakana, bcHangul |